简单记录一下我爬取推文的过程

# -*- coding: UTF-8 -*-
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException
from time import sleep
import json
import datetime


# edit these three variables
user = 'shiraishi_haruk'
start = datetime.datetime(2017, 1, 14)  # year, month, day
end = datetime.datetime(2017, 2, 3)  # year, month, day

# only edit these if you're having problems
delay = 3  # time to wait on each page load before reading the page
driver = webdriver.Chrome()  # options are Chrome() Firefox() Safari()


# don't mess with this stuff
twitter_ids_filename = 'all_ids.json'
days = (end - start).days + 1
id_selector = '.time a.tweet-timestamp'
tweet_selector = 'li.js-stream-item'
user = user.lower()
ids = []

def format_day(date):
    day = '0' + str(date.day) if len(str(date.day)) == 1 else str(date.day)
    month = '0' + str(date.month) if len(str(date.month)) == 1 else str(date.month)
    year = str(date.year)
    return '-'.join([year, month, day])

def form_url(since, until):
    p1 = 'https://twitter.com/search?f=tweets&vertical=default&q=from%3A'
    p2 =  user + '%20since%3A' + since + '%20until%3A' + until + 'include%3Aretweets&src=typd'
    return p1 + p2

def increment_day(date, i):
    return date + datetime.timedelta(days=i)


driver.get('https://twitter.com/search?f=tweets&vertical=default&q=from%3Ashiraishi_haruk%20since%3A2016-03-15%20until%3A2016-03-16include%3Aretweets&src=typd')
driver.find_element_by_name("session[username_or_email]").send_keys('youremail')
driver.find_element_by_name("session[password]").send_keys('yourpassword')
driver.find_element_by_class_name("submit").click()
sleep(30)
    
for day in range(days):
    d1 = format_day(increment_day(start, 0))
    d2 = format_day(increment_day(start, 1))
    url = form_url(d1, d2)
    print(url)
    print(d1)
    driver.get(url)
    
    sleep(delay)

    try:
        found_tweets = driver.find_elements_by_css_selector(tweet_selector)
        increment = 10

        while len(found_tweets) >= increment:
            print('scrolling down to load more tweets')
            driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
            sleep(delay)
            found_tweets = driver.find_elements_by_css_selector(tweet_selector)
            increment += 10

        print('{} tweets found, {} total'.format(len(found_tweets), len(ids)))

        for tweet in found_tweets:
            try:
                id = tweet.find_element_by_css_selector(id_selector).get_attribute('href').split('/')[-1]
                ids.append(id)
            except StaleElementReferenceException as e:
                print('lost element reference', tweet)

    except NoSuchElementException:
        print('no tweets on this day')

    start = increment_day(start, 1)


try:
    with open(twitter_ids_filename) as f:
        all_ids = ids + json.load(f)
        data_to_write = list(set(all_ids))
        print('tweets found on this scrape: ', len(ids))
        print('total tweet count: ', len(data_to_write))
except FileNotFoundError:
    with open(twitter_ids_filename, 'w') as f:
        all_ids = ids
        data_to_write = list(set(all_ids))
        print('tweets found on this scrape: ', len(ids))
        print('total tweet count: ', len(data_to_write))

with open(twitter_ids_filename, 'w') as outfile:
    json.dump(data_to_write, outfile)

print('all done here')
driver.close()

tw = twarc.Twarc(CONSUMER_KEY, CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
f=open("shiraishi_haruk_1701.json","w")
for id_no in ids:         # store the ids in this list
	try:
        tweet = tw.get('https://api.twitter.com/1.1/statuses/show/%s.json' % int(id_no))
        json.dump(tweet.json(),f)
    except Exception as e:
        print(e)
        time.sleep(60 * 15)
        continue
    except StopIteration:
        break
f.close()

import json
lines=[]
name='shiraishi_haruk_1604_l'
with open(name+'.json','r') as f:
	while True:
		line=f.readline()
		if not line: break
		line_json=json.loads(line)
		lines.append(line_json)

lines=sorted(lines,key=lambda k:k["id"],reverse=True)
with open(name+'_sorted.json','w') as fi:
	for item in lines:
		json.dump(item,fi)
		fi.write("\n")