-
Notifications
You must be signed in to change notification settings - Fork 0
/
fillDB.py
99 lines (76 loc) · 3.14 KB
/
fillDB.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import os
import threading
import time
import sys
import datetime
from airtable import Airtable
import logging
logging.basicConfig(format='%(asctime)s %(filename)s %(levelname)s: %(message)s',
level=logging.INFO)
sys.path.insert(1, os.path.join(sys.path[0], '/home/cmsdaq/AutoProcess'))
from airtableDB import *
#base_key = 'appQ2YoOIQFBKKIpG'
tables = ['RUNS','Crystals']
airtables={}
for t in tables:
airtables[t] = Airtable(base_key, t, api_key=os.environ['AIRTABLE_KEY'])
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--db',dest='db')
parser.add_argument('--start',dest='start')
parser.add_argument('--end',dest='end')
args = parser.parse_args()
print('Querying db %s for date %s'%(base_key,args.start.replace('_','-')))
import measDB
f=measDB.MeasDB(args.db)
runs=airtables['RUNS'].get_all()
xtals=airtables['Crystals'].get_all()
if (args.start == None):
start=datetime.datetime.now().replace(minute=0, hour=0, second=0)
else:
start=datetime.datetime.strptime(args.start.replace('_','-'),'%Y-%m-%d')
if (args.end == None):
end=start + datetime.timedelta(days=1)
else:
end=datetime.datetime.strptime(args.end,'%Y-%m-%d')
logging.info('Looking for runs from %s to %s'%(start.date(),end.date()))
from collections import defaultdict
ledRuns=defaultdict(list)
refRuns=defaultdict(list)
for r in runs:
if r['fields']['Processing status'] != 'VALIDATED':
continue
tR=datetime.datetime.strptime(r['fields']['Created'],'%Y-%m-%dT%H:%M:%S.%fZ')
if (tR<start or tR>end):
continue
if(r['fields']['Type']=='LED' and r['fields']['TAG']=='LED_DAILY'):
ledRuns[tR.date()].append(r['fields']['RunID'])
if(r['fields']['Type']=='SOURCE' and r['fields']['TAG']=='REF_DAILY'):
refRuns[tR.date()].append(r['fields']['RunID'])
measurements=defaultdict(dict)
for r in runs:
if r['fields']['Processing status'] != 'VALIDATED':
continue
tR=datetime.datetime.strptime(r['fields']['Created'],'%Y-%m-%dT%H:%M:%S.%fZ')
if (tR<start or tR>end):
continue
if(r['fields']['Type']=='SOURCE' and not r['fields']['TAG']=='REF_DAILY'):
xtalID=next((x['fields']['ID'] for x in xtals if x['id'] == (r['fields']['Crystal'])[0]), None)
if (xtalID is None):
logging.error('Xtal not found in Crystals table')
continue
if (xtalID == 'REF'):
logging.warning('Skip REF measurement %s not tagged as REF_DAILY'%r['fields']['RunID'])
continue
if (not tR.date() in measurements[xtalID].keys()):
measurements[xtalID][tR.date()]=[{'id':r['fields']['RunID'],'tag':r['fields']['TAG']}]
else:
measurements[xtalID][tR.date()].append({'id':r['fields']['RunID'],'tag':r['fields']['TAG']})
for xt,runs in measurements.items():
prod,geo=next(( ['prod%d'%x['fields']['VendorID'],x['fields']['Type'].lower()] for x in xtals if x['fields']['ID'] == xt), None)
for day,rr in runs.items():
runs=[ r['id'] for r in rr ]
xtalID=int((xt.split('BAR'))[1])
tag=rr[0]['tag']
f.insertMeas(xt,prod,geo,xtalID,runs,refRuns[day],ledRuns[day],tag)
f.save()