forked from ConservationInternational/aichi1
-
Notifications
You must be signed in to change notification settings - Fork 0
/
exportCSV.py
27 lines (21 loc) · 870 Bytes
/
exportCSV.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
import pandas as pd
from pymongo import MongoClient
import boto3
s3 = boto3.resource('s3')
client = MongoClient('localhost', 27017)
countries = pd.read_csv('/home/ec2-user/aichi1/countries.csv', names=['fullname', 'country'], na_values="", keep_default_na=False)
def writeDfToS3Csv(df, name):
s3.Bucket('ci-tweet-csv-dumps').put_object(Key=name + '.csv', Body=df.to_csv(index=False), ACL='public-read')
def getDfFromMongo(conname):
temp = []
for i in client.TWITTER[c].find():
temp.append(i)
df = pd.DataFrame(temp)
del df['_id']
return df
for c in ['TRENDS', 'TWITTER-BASELINE', 'TWITTER-DETAIL', 'WEBHOSE-BASELINE', 'WEBHOSE-DETAIL']:
df = getDfFromMongo(c)
df = pd.merge(df, countries)
df = df[df['month'] != '2017-10']
writeDfToS3Csv(df, c)
df.to_csv("~/aichi1/myapp/public/csvs/" + c + ".csv", index=False)