Skip to content

Commit 7134ebf

Browse files
vpet98vpet98
committed
Break tasks
1 parent f040ba7 commit 7134ebf

4 files changed

Lines changed: 178 additions & 377 deletions

File tree

‎board/tasks.py‎

Lines changed: 21 additions & 240 deletions
Original file line numberDiff line numberDiff line change
@@ -1,247 +1,28 @@
1-
import logging
2-
from datetime import datetime
3-
from urllib.parse import urlparse
4-
5-
import requests
6-
from celery import shared_task
7-
from django.db.models import Q
1+
from celery import shared_task, chain
82
from django.utils import timezone
93

10-
from .views import update_datacite_table, update_stationxml_table
11-
from .models import (
12-
Consistency,
13-
Datacenter,
14-
Datacite,
15-
Eida_routing,
16-
Fdsn_registry,
17-
Stationxml,
4+
from .views import (
5+
get_FDSN_networks,
6+
get_FDSN_datacenters,
7+
get_EIDA_routing,
8+
consistency_from_FDSN,
9+
consistency_from_xml,
10+
consistency_from_routing
1811
)
1912

20-
logger = logging.getLogger(__name__)
21-
22-
2313
@shared_task
24-
def update_and_run():
25-
logger.info("Getting all networks from FDSN and updating database")
26-
r = requests.get("https://www.fdsn.org/ws/networks/1/query", timeout=20)
27-
for net in r.json()["networks"]:
28-
doi = net["doi"] if net["doi"] else None
29-
start = datetime.strptime(net["start_date"], "%Y-%m-%d").astimezone(
30-
datetime.timezone.utc
31-
)
32-
end = (
33-
datetime.strptime(net["end_date"], "%Y-%m-%d").astimezone(
34-
datetime.timezone.utc
35-
)
36-
if net["end_date"]
37-
else None
38-
)
39-
obj, created = Fdsn_registry.objects.update_or_create(
40-
netcode=net["fdsn_code"],
41-
startdate=start,
42-
defaults={"enddate": end, "doi": doi},
43-
)
44-
if net["doi"] != "":
45-
update_datacite_table(obj)
46-
47-
logger.info("Getting all datacenters from FDSN and updating database")
48-
r = requests.get("https://www.fdsn.org/ws/datacenters/1/query", timeout=20)
49-
for dc in r.json()["datacenters"]:
50-
station_url = None
51-
for r in dc["repositories"]:
52-
for s in r["services"]:
53-
if s["name"] == "fdsnws-station-1":
54-
station_url = s["url"]
55-
datacenter = Datacenter(
56-
name=dc["name"],
57-
station_url=urlparse(station_url).netloc
58-
if station_url is not None
59-
else None,
60-
)
61-
datacenter.save()
62-
if dc["name"] in [
63-
"GEOFON",
64-
"ODC",
65-
"ETH",
66-
"RESIF",
67-
"INGV",
68-
"LMU",
69-
"ICGC",
70-
"NOA",
71-
"BGR",
72-
"NIEP",
73-
"KOERI",
74-
"UIB-NORSAR",
75-
]:
76-
update_stationxml_table(datacenter)
14+
def update_db():
15+
get_FDSN_networks()
16+
get_FDSN_datacenters()
17+
get_EIDA_routing()
7718

78-
logger.info("Getting EIDA routing information")
79-
r = requests.get(
80-
"https://www.orfeus-eu.org/eidaws/routing/1/query?format=json&service=station",
81-
timeout=20,
82-
)
83-
for dc in r.json():
84-
# get datacenter from database
85-
datacenter = Datacenter.objects.get(station_url=urlparse(dc["url"]).netloc)
86-
for net in dc["params"]:
87-
try:
88-
net_start = (
89-
datetime.strptime(net["start"], "%Y-%m-%dT%H:%M:%S")
90-
.astimezone(datetime.timezone.utc)
91-
.date()
92-
)
93-
except Exception:
94-
net_start = (
95-
datetime.strptime(net["start"], "%Y-%m-%dT%H:%M:%S.%f")
96-
.astimezone(datetime.timezone.utc)
97-
.date()
98-
)
99-
try:
100-
net_end = (
101-
datetime.strptime(net["end"], "%Y-%m-%dT%H:%M:%S")
102-
.astimezone(datetime.timezone.utc)
103-
.date()
104-
)
105-
except Exception:
106-
try:
107-
net_end = (
108-
datetime.strptime(net["end"], "%Y-%m-%dT%H:%M:%S.%f")
109-
.astimezone(datetime.timezone.utc)
110-
.date()
111-
)
112-
except Exception:
113-
net_end = None
114-
Eida_routing.objects.update_or_create(
115-
netcode=net["net"],
116-
datacenter=datacenter,
117-
startdate=net_start,
118-
defaults={"enddate": net_end, "priority": net["priority"]},
119-
)
120-
logger.info("Finished updating DB from sources")
121-
122-
logger.info("Checking consistency for each available network")
19+
@shared_task
20+
def tests_run():
12321
current_time = timezone.now().replace(microsecond=0)
124-
# starting from Fdsn_registry table
125-
logger.info("Making consistency checks starting from FDSN registry")
126-
for net in Fdsn_registry.objects.all():
127-
fdsn_end = (
128-
net.enddate
129-
if net.enddate is not None
130-
else datetime.strptime("2100-01-01", "%Y-%m-%d")
131-
.astimezone(datetime.timezone.utc)
132-
.date()
133-
)
134-
routings = Eida_routing.objects.filter(
135-
netcode=net.netcode, startdate__range=(net.startdate, fdsn_end)
136-
)
137-
datacite = Datacite.objects.filter(network=net).first()
138-
if datacite is None:
139-
page_works, has_license = None, None
140-
else:
141-
try:
142-
r = requests.get(datacite.page, timeout=10)
143-
page_works = bool(r.status_code == 200)
144-
except requests.exceptions.RequestException:
145-
page_works = False
146-
has_license = bool(datacite.licenses is not None)
147-
if not routings.exists():
148-
xmls = Stationxml.objects.filter(
149-
netcode=net.netcode, startdate__range=(net.startdate, fdsn_end)
150-
)
151-
if xmls.exists():
152-
for xml in xmls:
153-
if net.doi is None:
154-
doi_match = None
155-
elif xml.doi is None:
156-
doi_match = False
157-
else:
158-
doi_match = net.doi.lower() == xml.doi.lower()
159-
Consistency(
160-
test_time=current_time,
161-
fdsn_net=net,
162-
xml_net=xml,
163-
doi=net.doi,
164-
page_works=page_works,
165-
has_license=has_license,
166-
xml_doi_match=doi_match,
167-
).save()
168-
else:
169-
Consistency(
170-
test_time=current_time,
171-
fdsn_net=net,
172-
doi=net.doi,
173-
page_works=page_works,
174-
has_license=has_license,
175-
).save()
176-
else:
177-
for rout in routings:
178-
rout_end = (
179-
rout.enddate
180-
if rout.enddate is not None
181-
else datetime.strptime("2100-01-01", "%Y-%m-%d")
182-
.astimezone(datetime.timezone.utc)
183-
.date()
184-
)
185-
start_search = min(rout.startdate, net.startdate)
186-
end_search = max(rout_end, fdsn_end)
187-
xml = Stationxml.objects.filter(
188-
datacenter=rout.datacenter.name,
189-
netcode=rout.netcode,
190-
startdate__range=(start_search, end_search),
191-
).first()
192-
if net.doi is None or xml is None:
193-
doi_match = None
194-
elif xml.doi is None:
195-
doi_match = False
196-
else:
197-
doi_match = net.doi.lower() == xml.doi.lower()
198-
Consistency(
199-
test_time=current_time,
200-
fdsn_net=net,
201-
eidarout_net=rout,
202-
xml_net=xml if xml is not None else None,
203-
doi=net.doi,
204-
page_works=page_works,
205-
has_license=has_license,
206-
xml_doi_match=doi_match,
207-
).save()
208-
# starting from Stationxml table
209-
logger.info("Making consistency checks starting from StationXML files")
210-
unlinked_stationxml = Stationxml.objects.filter(
211-
~Q(
212-
id__in=Consistency.objects.filter(test_time=current_time).values_list(
213-
"xml_net_id", flat=True
214-
)
215-
)
216-
)
217-
for net in unlinked_stationxml:
218-
routings = Eida_routing.objects.filter(
219-
Q(
220-
netcode=net.netcode,
221-
startdate__lte=net.startdate,
222-
enddate__gte=net.startdate,
223-
)
224-
| Q(
225-
netcode=net.netcode,
226-
startdate__lte=net.startdate,
227-
enddate__isnull=True,
228-
)
229-
)
230-
if not routings.exists():
231-
Consistency(test_time=current_time, xml_net=net, doi=net.doi).save()
232-
for rout in routings:
233-
Consistency(
234-
test_time=current_time, xml_net=net, eidarout_net=rout, doi=net.doi
235-
).save()
236-
# starting from Eida_routing table
237-
logger.info("Making consistency checks starting from EIDA routing registry")
238-
unlinked_routing = Eida_routing.objects.filter(
239-
~Q(
240-
id__in=Consistency.objects.filter(test_time=current_time).values_list(
241-
"eidarout_net_id", flat=True
242-
)
243-
)
244-
)
245-
for net in unlinked_routing:
246-
Consistency(test_time=current_time, eidarout_net=net).save()
247-
logger.info("Finished running tests for all networks")
22+
consistency_from_FDSN(current_time)
23+
consistency_from_xml(current_time)
24+
consistency_from_routing(current_time)
25+
26+
@shared_task
27+
def update_db_and_run_tests():
28+
chain(update_db.si(), tests_run.si()).apply_async()

0 commit comments

Comments
 (0)