-
Notifications
You must be signed in to change notification settings - Fork 0
/
trawl_vizier.py
69 lines (53 loc) · 2.5 KB
/
trawl_vizier.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
from selenium import webdriver
import time
VizieR_ROOT = 'https://vizier.u-strasbg.fr/viz-bin/'
CG_PATH = 'VizieR-3?-source=J/A%2bA/618/A93/table1'
OPTIONS = '&-out.max=50&-out.form=HTML%20Table&-out.add=_r&-out.add=_RAJ,_DEJ&-sort=_r&-oc.form=sexa'
CLUSTER_ROOT = 'https://vizier.u-strasbg.fr/viz-bin/VizieR-6?-out.form=+H&-source=J/A+A/618/A93&Cluster='
CLUSTER_PLOT1 = 'http://cdsportal.u-strasbg.fr/widgets/dataplot/dataplot.html?dataset_url_1=&dataset_url_1=https%3A%2F%2Fvizier.u-strasbg.fr%2Fviz-bin%2Fvotable%3F-ref%3DVIZ5fed9b736a7e%26%2F%2Foutaddvalue%3Ddefault%26-order%3DI%26-oc.form%3Dsexa%26-c.r%3D%2520%25202%26-c.geom%3Dr%26-source%3DJ%252FA%252BA%252F618%252FA93%26Cluster%3D'
CLUSTER_PLOT2 = '%26-meta.ucd%3D2%26-meta%3D1%26-meta.foot%3D1%26-out.max%3Dunlimited%26%3DCDS%252C%2520France%26-c.eq%3DJ2000%26-c.u%3Darcmin&option_graph_title=%20J/A+A/618/A93'
# Open VizieR website
driver = webdriver.Firefox()
driver.get(VizieR_ROOT+CG_PATH+OPTIONS)
# Submit request for catalogue
driver.find_element_by_xpath("//select[@name='-out.max']/option[text()='unlimited']").click()
element = driver.find_element_by_xpath(".//*[@id='vcst']")
element.submit()
time.sleep(30)
# Make a list of all the clusters from the table
clusterlist = []
x = 0
table = driver.find_element_by_css_selector("#c36180093t1_1")
for row in table.find_elements_by_css_selector('tr'):
x = 0
for d in row.find_elements_by_css_selector('td'):
x += 1
if (x == 4):
clusterlist.append(d.text)
break
i = 0
error_clusters = []
for clustername in clusterlist:
i += 1
clusterlink = CLUSTER_PLOT1 + clustername + CLUSTER_PLOT2
driver.get(clusterlink)
print(f'Working on {clustername}, {i}/{len(clusterlist)}')
time.sleep(10)
for _ in range(10):
try:
driver.find_element_by_tag_name('canvas')
except:
if _ == 9:
print(f'Cluster {clustername} without a proper file, added to error list')
error_clusters.append(clustername+'\n')
break
time.sleep(2)
print(f'Cluster {clustername} slow to load, trying again')
else:
driver.save_screenshot('/home/sbalan7/Desktop/Code/charting-Gaia/plots/' + clustername + '.png')
print(f'Saved a file for cluster {clustername}')
break
print(f'Failed to gather plots for {len(error_clusters)} images, writing error file')
with open('failed.txt', 'w') as f:
f.writelines(error_clusters)
driver.quit()