From f496bc5efa09960e69d26c2cd8d1f9d607461e5b Mon Sep 17 00:00:00 2001
From: Patrick Westerhoff <PatrickWesterhoff@gmail.com>
Date: Sat, 24 Oct 2015 18:26:32 +0200
Subject: [PATCH] Add avatar fetcher

---
 .gitignore          |  2 ++
 README.md           | 29 ++++++++++++++++
 avatar-fetcher.py   | 81 +++++++++++++++++++++++++++++++++++++++++++++
 config.example.json |  4 +++
 4 files changed, 116 insertions(+)
 create mode 100644 README.md
 create mode 100644 avatar-fetcher.py
 create mode 100644 config.example.json

diff --git a/.gitignore b/.gitignore
index e69de29..b8318e1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/config.json
+/output
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..71331ce
--- /dev/null
+++ b/README.md
@@ -0,0 +1,29 @@
+# Twitter Avatar Fetcher
+This small utility script fetches user avatars from Twitter.
+
+## Setup
+1. Register a Twitter app at https://apps.twitter.com (see below).
+2. Copy the `config.example.json` to `config.json`.
+3. Fill in the `consumer_key` and `consumer_secret` values from the “Keys and Access Tokens” tab of your Twitter app.
+
+## Usage
+In order to run the program, you need Python 3.
+
+Place the screen names of the users you want to fetch the avatars from into a text file, with one screen name per line. Then run the the avatar fetcher like this:
+
+    ./avatar-fetcher.py user-file.txt
+
+By default, the program will place all files into an `output` folder. You can modify that with the `-t` command line parameter. For full help, run the script with the `-h` flag.
+
+## Registering a Twitter app
+If you haven’t already, you need to add your mobile phone number to your Twitter account. This is unfortunately required to register apps now.
+
+1. Go to https://apps.twitter.com/app/new.
+2. Enter the required fields “Name”, “Description”, and “Website”. The values don’t matter as the app is never used for direct user interaction, but you should not enter random garbage either in case Twitter checks these values at some point. You could for example use the following values:
+   * Name: “Twitter Avatar Fetcher”
+   * Description: “Fetches avatars from Twitter”
+   * Website: Your own website
+3. Leave the “Callback URL” field blank.
+4. Accept the developer agreement and click the “Create your Twitter application” button.
+5. Optional: On the “Permissions” tab of your created app, change the mode to “Read-only”.
+6. The keys are accessible from the “Keys and Access Tokens” tab.
diff --git a/avatar-fetcher.py b/avatar-fetcher.py
new file mode 100644
index 0000000..6ce1703
--- /dev/null
+++ b/avatar-fetcher.py
@@ -0,0 +1,81 @@
+#! /usr/bin/env python3
+from urllib.request import urlopen, Request
+import argparse
+import base64
+import json
+import os
+import re
+import shutil
+import urllib.parse
+
+reScreenName = re.compile('^[a-z0-9_]{1,15}$', re.I)
+def validateScreenName (screenName):
+	if reScreenName.match(screenName):
+		return True
+
+	print('Invalid screen name:', screenName.encode())
+	return False
+
+class TwitterAvatarFetcher:
+	def __init__ (self, targetFolder):
+		if not os.path.exists(targetFolder):
+			os.makedirs(targetFolder)
+		self.targetFolder = targetFolder
+
+		with open('config.json') as f:
+			config = json.load(f)
+		self.key = base64.b64encode('{}:{}'.format(config['consumer_key'], config['consumer_secret']).encode()).decode()
+
+	def retrieveBearerToken (self):
+		req = Request('https://api.twitter.com/oauth2/token')
+		req.method = 'POST'
+		req.add_header('Authorization', 'Basic {}'.format(self.key))
+		req.add_header('Content-Type', 'application/x-www-form-urlencoded;charset=utf-8')
+		req.data = b'grant_type=client_credentials'
+
+		with urlopen(req) as resp:
+			data = json.loads(resp.read().decode())
+
+		self.bearerToken = data['access_token']
+
+	def getUsers (self, users):
+		req = Request('https://api.twitter.com/1.1/users/lookup.json')
+		req.method = 'POST'
+		req.data = urllib.parse.urlencode({ 'screen_name': ','.join(users) }).encode()
+		req.add_header('Authorization', 'Bearer {}'.format(self.bearerToken))
+		req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+		with urlopen(req) as resp:
+			return json.loads(resp.read().decode())
+
+	def downloadAvatars (self, users):
+		users = list(filter(validateScreenName, users))
+		for i in range(0, len(users), 50):
+			for user in self.getUsers(users[i:i+50]):
+				screenName = user['screen_name']
+				url = user['profile_image_url'].replace('_normal', '')
+				path = os.path.join(self.targetFolder, screenName + os.path.splitext(url)[1])
+				try:
+					with urlopen(url) as res, open(path, 'wb+') as f:
+						shutil.copyfileobj(res, f)
+				except Exception as e:
+					print('Download failed for "{}"\n  '.format(screenName, e))
+
+def main ():
+	parser = argparse.ArgumentParser(description='Fetch user avatars from Twitter.')
+	parser.add_argument('file', help='File with usernames')
+	parser.add_argument('-t', '--target', default='output', help='target path')
+	args = parser.parse_args()
+
+	if not os.path.exists('config.json'):
+		parser.error('You need to create a config.json file first. Check the README file for details.')
+
+	dl = TwitterAvatarFetcher(args.target)
+	dl.retrieveBearerToken()
+
+	with open(args.file, encoding='utf-8') as f:
+		users = list(line.strip() for line in f)
+	dl.downloadAvatars(users)
+
+if __name__ == '__main__':
+	main()
diff --git a/config.example.json b/config.example.json
new file mode 100644
index 0000000..4e855fa
--- /dev/null
+++ b/config.example.json
@@ -0,0 +1,4 @@
+{
+	"consumer_key": "",
+	"consumer_secret": ""
+}