-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.py
86 lines (69 loc) · 2.43 KB
/
app.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import tweepy
import config
import logging
import shelve
import time
from contextlib import closing
# from collections import OrderedDict
logging.basicConfig(filename='archiver.log',
level=getattr(logging, config.LOG_LEVEL))
AUTH = tweepy.OAuthHandler(config.API_KEY, config.API_SECRET)
AUTH.set_access_token(config.ACCESS_TOKEN, config.ACCESS_KEY)
API = tweepy.API(auth_handler=AUTH)
class StreamListener(tweepy.StreamListener):
def __init__(self, store_path, *args, **kwargs):
self._store_path = store_path
super(StreamListener, self).__init__(*args, **kwargs)
def on_status(self, status):
# write each status in a new file
# write each attached image in a new file
print status.text
TweetStore(self._store_path, user=status.user).add(status)
def on_error(self, status_code):
print "error from twitter API {}".format(status_code)
time.sleep(2)
return True
class TweetStore(object):
def __init__(self, store_path, user, api=None):
assert isinstance(user, tweepy.models.User)
self._tweets = []
self._user = user
self._store_path = store_path
self.api = api
self._user = user
def save(self):
with closing(shelve.open(self._store_path)) as store:
store[str(self._user.id_str)] = self._tweets
def add(self, tweet):
self._tweets.append(tweet)
self.save()
def update(self):
new_tweets = self.api.user_timeline(
user_id=self.user.id,
since_id=self._tweets[-1].id if self._tweets else None
)
new_tweets.reverse()
for _ in new_tweets:
self.add(_)
self.save()
@property
def tweets(self):
return self._tweets
@property
def user(self):
return self._user
def run():
shelf_path = 'user_tweets_store.shelf'
listener = StreamListener(shelf_path)
streamer = tweepy.Stream(auth=AUTH,
listener=listener)
user_ids_to_follow = []
for screen_name in config.SCREEN_NAMES_TO_FOLLOW:
user = API.get_user(screen_name)
TweetStore(user=user, store_path=shelf_path, api=API).update()
user_ids_to_follow.append(user.id_str)
print "following IDs: " + ", ".join(user_ids_to_follow)
print "Listening for statuses"
streamer.filter(follow=user_ids_to_follow, async=True)
if __name__ == '__main__':
run()