Add avatar fetcher

.gitignore

@@ -0,0 +1,2 @@
+/config.json
+/output

README.md

@@ -0,0 +1,29 @@
+# Twitter Avatar Fetcher
+This small utility script fetches user avatars from Twitter.
+
+## Setup
+1. Register a Twitter app at https://apps.twitter.com (see below).
+2. Copy the `config.example.json` to `config.json`.
+3. Fill in the `consumer_key` and `consumer_secret` values from the “Keys and Access Tokens” tab of your Twitter app.
+
+## Usage
+In order to run the program, you need Python 3.
+
+Place the screen names of the users you want to fetch the avatars from into a text file, with one screen name per line. Then run the the avatar fetcher like this:
+
+    ./avatar-fetcher.py user-file.txt
+
+By default, the program will place all files into an `output` folder. You can modify that with the `-t` command line parameter. For full help, run the script with the `-h` flag.
+
+## Registering a Twitter app
+If you haven’t already, you need to add your mobile phone number to your Twitter account. This is unfortunately required to register apps now.
+
+1. Go to https://apps.twitter.com/app/new.
+2. Enter the required fields “Name”, “Description”, and “Website”. The values don’t matter as the app is never used for direct user interaction, but you should not enter random garbage either in case Twitter checks these values at some point. You could for example use the following values:
+   * Name: “Twitter Avatar Fetcher”
+   * Description: “Fetches avatars from Twitter”
+   * Website: Your own website
+3. Leave the “Callback URL” field blank.
+4. Accept the developer agreement and click the “Create your Twitter application” button.
+5. Optional: On the “Permissions” tab of your created app, change the mode to “Read-only”.
+6. The keys are accessible from the “Keys and Access Tokens” tab.

avatar-fetcher.py

@@ -0,0 +1,81 @@
+#! /usr/bin/env python3
+from urllib.request import urlopen, Request
+import argparse
+import base64
+import json
+import os
+import re
+import shutil
+import urllib.parse
+
+reScreenName = re.compile('^[a-z0-9_]{1,15}$', re.I)
+def validateScreenName (screenName):
+	if reScreenName.match(screenName):
+		return True
+
+	print('Invalid screen name:', screenName.encode())
+	return False
+
+class TwitterAvatarFetcher:
+	def __init__ (self, targetFolder):
+		if not os.path.exists(targetFolder):
+			os.makedirs(targetFolder)
+		self.targetFolder = targetFolder
+
+		with open('config.json') as f:
+			config = json.load(f)
+		self.key = base64.b64encode('{}:{}'.format(config['consumer_key'], config['consumer_secret']).encode()).decode()
+
+	def retrieveBearerToken (self):
+		req = Request('https://api.twitter.com/oauth2/token')
+		req.method = 'POST'
+		req.add_header('Authorization', 'Basic {}'.format(self.key))
+		req.add_header('Content-Type', 'application/x-www-form-urlencoded;charset=utf-8')
+		req.data = b'grant_type=client_credentials'
+
+		with urlopen(req) as resp:
+			data = json.loads(resp.read().decode())
+
+		self.bearerToken = data['access_token']
+
+	def getUsers (self, users):
+		req = Request('https://api.twitter.com/1.1/users/lookup.json')
+		req.method = 'POST'
+		req.data = urllib.parse.urlencode({ 'screen_name': ','.join(users) }).encode()
+		req.add_header('Authorization', 'Bearer {}'.format(self.bearerToken))
+		req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+		with urlopen(req) as resp:
+			return json.loads(resp.read().decode())
+
+	def downloadAvatars (self, users):
+		users = list(filter(validateScreenName, users))
+		for i in range(0, len(users), 50):
+			for user in self.getUsers(users[i:i+50]):
+				screenName = user['screen_name']
+				url = user['profile_image_url'].replace('_normal', '')
+				path = os.path.join(self.targetFolder, screenName + os.path.splitext(url)[1])
+				try:
+					with urlopen(url) as res, open(path, 'wb+') as f:
+						shutil.copyfileobj(res, f)
+				except Exception as e:
+					print('Download failed for "{}"\n  '.format(screenName, e))
+
+def main ():
+	parser = argparse.ArgumentParser(description='Fetch user avatars from Twitter.')
+	parser.add_argument('file', help='File with usernames')
+	parser.add_argument('-t', '--target', default='output', help='target path')
+	args = parser.parse_args()
+
+	if not os.path.exists('config.json'):
+		parser.error('You need to create a config.json file first. Check the README file for details.')
+
+	dl = TwitterAvatarFetcher(args.target)
+	dl.retrieveBearerToken()
+
+	with open(args.file, encoding='utf-8') as f:
+		users = list(line.strip() for line in f)
+	dl.downloadAvatars(users)
+
+if __name__ == '__main__':
+	main()

config.example.json

@@ -0,0 +1,4 @@
+{
+	"consumer_key": "",
+	"consumer_secret": ""
+}