-
Notifications
You must be signed in to change notification settings - Fork 1
/
TraningData.py
56 lines (43 loc) · 1.88 KB
/
TraningData.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import sqlite3
import pandas as pd
timeframes = ['2018-05']
for timeframe in timeframes:
connection = sqlite3.connect('{}.db'.format(timeframe))
cursor = connection.cursor()
limit = 5000
last_unix = 0
cur_length = limit
counter = 0
test_done2012 = False
test_done2013 = False
while cur_length == limit:
datafile = pd.read_sql("SELECT * FROM parent_reply WHERE unix > {} and parent NOT NULL and score > 0 ORDER BY unix ASC LIMIT {}".format(last_unix,limit),connection)
last_unix = datafile.tail(1)['unix'].values[0]
cur_length = len(datafile)
if not test_done2013:
with open('tst2013.from','a', encoding='utf8') as file:
for content in datafile['parent'].values:
file.write(content+'\n')
with open('tst2013.to','a', encoding='utf8') as file:
for content in datafile['comment'].values:
file.write(str(content)+'\n')
test_done2013 = True
if not test_done2012:
with open('tst2012.from','a', encoding='utf8') as file:
for content in datafile['parent'].values:
file.write(content+'\n')
with open('tst2012.to','a', encoding='utf8') as file:
for content in datafile['comment'].values:
file.write(str(content)+'\n')
test_done2012 = True
else:
with open('train.from','a', encoding='utf8') as file:
for content in datafile['parent'].values:
file.write(content+'\n')
with open('train.to','a', encoding='utf8') as file:
for content in datafile['comment'].values:
file.write(str(content)+'\n')
counter += 1
if counter % 20 == 0:
print(counter*limit,'rows completed so far')
print("you re good to go!")