forked from ReagentX/HLTVDemoDownloader
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Downloader.py
executable file
·215 lines (160 loc) · 6.62 KB
/
Downloader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
from multiprocessing.dummy import Pool as ThreadPool
import urllib2
import urllib
import re
import os
def get_match_ids(eventid):
# Create an offset variable for lists that are paginated on HLTV
offset = 0
# Build the URL
# Create an array of all of the Demo URLs on the page
match_ids = find_match_ids_at_url("https://www.hltv.org/results?offset=%s&event=%s" % (offset, eventid))
# If the length is = 50, offset by 50 and loop again
if len(match_ids) == 50:
print "Parsed first page. Found %s IDs" % (len(match_ids))
# Set a boolean to close the while loop and a page variable we can increment when paginating
more_pages = True
page = 1
# While check is true, offset by 50
while more_pages:
offset += 50
# Same URL building and parsing as above
more_match_ids = find_match_ids_at_url("https://www.hltv.org/results?offset=%s&event=%s" % (offset, eventid))
for match in more_match_ids:
match_ids.append(match)
# Determine if there are additional pages to be found, if not the while loop ends
if len(more_match_ids) < 50:
more_pages = False
page += 1
print "Parsed page %s. Found %s IDs." % (page, len(match_ids))
else:
# Prints the current page and the number of parsed IDs
page += 1
print "Parsed page %s. %s IDs found so far." % (page, len(match_ids))
elif len(match_ids) < 50:
print "Total demos: %s" % len(match_ids)
elif len(match_ids) > 50:
print "HLTV altered demo page layout :("
return match_ids
def find_match_ids_at_url(url):
# Get the HTML using get_html()
html = get_html(url)
# Create an array of all of the Demo URLs on the page
match_ids = re.findall('<div class="result-con"><a href=\"/matches/(.*?)\"', html)
return match_ids
def convert_to_demo_ids(match_ids, threads):
# Tell the user what is happening
print "Converting Match IDs to Demo IDs"
# Define the number of threads
pool = ThreadPool(threads)
# Calls get_demo_ids() and adds the value returned each call to an array called demo_ids
demo_ids = pool.map(get_demo_ids, match_ids)
pool.close()
pool.join()
# Create an array to add any captured errors to
errors = []
# Find any errors, add them to the errors array, and remove them from demo_ids
for demo_id in demo_ids:
if "/" in demo_id:
errors.append(demo_id)
demo_ids = [x for x in demo_ids if x not in errors]
# Print the errors (if there are any)
print_errors(errors)
return demo_ids
def get_demo_ids(match_id):
# URL building and opening
url = "https://www.hltv.org/matches/%s" % (match_id)
html = get_html(url)
demo_id = re.findall('"/download/demo/(.*?)"', html)
# Check if re.findall()'s array is empty
# If it has an element, add that Demo ID to the demo_ids array
if len(demo_id) > 0:
# Loop through the demo_ids array and remove everything up to the last / to get the real Demo ID
for i in range(0, len(demo_id)):
print "Converted %s" % (match_id)
# Return the Demo ID
return demo_id[0]
# If there is no element, print which match has no demo
elif len(demo_id) < 1:
print "No demo found for %s" % (match_id)
# Return the Match ID with a space char so we can find it later
return " %s" % match_id
def download(demo_ids, threads=1):
# Temporarily use 1 due to 503 errors
# Convert the DemoIDs to URLs
urls = convert_to_urls(demo_ids)
# Define the number of threads
pool = ThreadPool(threads)
# Make a folder for the event to save the files in
directory = make_dir()
# Calls get() and adds the filesize returned each call to an array called filesizes
filesizes = pool.map(get, urls)
pool.close()
pool.join()
# Create a float to store the filesizes in and add them together
total_file_size = sum(filesizes)
# Print the properly formatted filesize.
print "Successfully transferred %s. Enjoy!" % (format_file_size(total_file_size))
return True
def convert_to_urls(demo_ids):
return ["https://www.hltv.org/download/demo/%s" % demo_id for demo_id in demo_ids]
def get(url):
# Build and open the URL
opener = urllib2.build_opener()
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
response = opener.open(url)
# HLTV redicrects to a .rar or .zip file
final_url = response.geturl()
# Gets the filename (everything after the last trailing /)
filename = final_url.rsplit('/', 1)[-1]
# Gets the Content-Length from the metadata from final_url
filesize = (int(urllib.urlopen(final_url).info().getheaders("Content-Length")[0])/1024)/1024
# Tell user we are downloading filesize
print "Starting %s: %s MB." % (filename, filesize)
# Downloads the file to the directory the user enters
urllib.urlretrieve(final_url, directory+"/"+filename)
# Tell user the current status and file information
print "Completed %s: %s MB." % (filename, filesize)
return filesize
def make_dir():
# Ask the user what the event name is
event_name = raw_input("What is the event name? ")
# Create a global variable so the different threads can access it
global directory
directory = "./%s" % (event_name)
os.mkdir(directory)
# Return the string so we can use it
return directory
def format_file_size(filesize):
if filesize > 1024:
return "%.2f GB" % (float(filesize) / 1024)
else:
return "%s MB" % (int(filesize))
def get_html(url):
# Open the URL
opener = urllib2.build_opener()
# Spoof the user agent
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
response = opener.open(url)
# Read the response as HTML
html = response.read()
return html
def print_errors(errors):
# Print URL(s) for the match(es) with no demo file(s)
if len(errors) == 1:
print "%s match has no demo:" % (len(errors))
for i in range(0, len(errors)):
print "%s: https://www.hltv.org/matches/%s" % (i+1, errors[i])
elif len(errors) > 0:
print "%s matches have no demo:" % (len(errors))
for i in range(0, len(errors)):
print "%s: https://www.hltv.org/matches/%s" % (i+1, errors[i])
else:
print "No errors found!"
return True
# Calls the method for a given Event ID.
event_id = raw_input("What is the event ID? ")
processes = 8
match_ids = get_match_ids(event_id)
demo_ids = convert_to_demo_ids(match_ids, processes)
download(demo_ids)