Switch to ChirpStack integration

Signed-off-by: Dimitar Dimitrov <[email protected]>

integration/init-db.py

@@ -20,23 +20,23 @@ def main():
     # than the GPS point coordinates table.
     cur.execute('CREATE TABLE reports('
                     'id INTEGER PRIMARY KEY AUTOINCREMENT,'
-                    'app_eui_id INTEGER,'
                     'dev_eui_id INTEGER,'
                     'dev_addr_id INTEGER,'
                     'dc_balance INTEGER,'
                     'fcnt INTEGER,'
                     'port INTEGER,'
                     'name_id INTEGER,'
+                    'profile_id INTEGER,'
                     'battery_voltage REAL,'
                     'reported_at_ms UNSIGNED BIGINT,'
-                    'FOREIGN KEY(app_eui_id) REFERENCES app_eui(id),'
                     'FOREIGN KEY(dev_eui_id) REFERENCES dev_eui(id),'
                     'FOREIGN KEY(dev_addr_id) REFERENCES dev_addr(id),'
-                    'FOREIGN KEY(name_id) REFERENCES device_names(id))')
+                    'FOREIGN KEY(name_id) REFERENCES device_names(id),'
+                    'FOREIGN KEY(profile_id) REFERENCES profile_names(id))')
     cur.execute('CREATE TABLE hotspot_connections('
                     'id INTEGER PRIMARY KEY AUTOINCREMENT,'
                     'report_id INTEGER NOT NULL,'
-                    'frequency REAL,'
+                    'frequency INTEGER,'
                     'name_id INTEGER,'
                     'rssi REAL,'
                     'snr REAL,'
@@ -50,28 +50,19 @@ def main():
     cur.execute('CREATE TABLE device_names('
                     'id INTEGER PRIMARY KEY AUTOINCREMENT,'
                     'name VARCHAR(128))')
+    cur.execute('CREATE TABLE profile_names('
+                    'id INTEGER PRIMARY KEY AUTOINCREMENT,'
+                    'name VARCHAR(128))')
     cur.execute('CREATE TABLE measurements('
                     'id INTEGER PRIMARY KEY AUTOINCREMENT,'
                     'report_id INTEGER NOT NULL,'
                     'temperature REAL,'
                     'pressure REAL,'
                     'humidity REAL,'
                     'FOREIGN KEY(report_id) REFERENCES reports(id))')
-    cur.execute('CREATE TABLE label_reports('
-                    'id INTEGER PRIMARY KEY AUTOINCREMENT,'
-                    'report_id INTEGER NOT NULL,'
-                    'name_id INTEGER NOT NULL,'
-                    'FOREIGN KEY(report_id) REFERENCES reports(id),'
-                    'FOREIGN KEY(name_id) REFERENCES label_strings(id))')
-    cur.execute('CREATE TABLE label_strings('
-                    'id INTEGER PRIMARY KEY AUTOINCREMENT,'
-                    'name VARCHAR(128))')
     # Store EUI as strings because SQLite3 Python
     # binding cannot handle 64-bit Python integers,
     # even when declaring columns as UNSIGNED BIGINT.
-    cur.execute('CREATE TABLE app_eui('
-                    'id INTEGER PRIMARY KEY AUTOINCREMENT,'
-                    'name VARCHAR(16))')
     cur.execute('CREATE TABLE dev_eui('
                     'id INTEGER PRIMARY KEY AUTOINCREMENT,'
                     'name VARCHAR(16))')

integration/meteo.py

@@ -15,6 +15,7 @@
 import json
 import struct
 import binascii
+import datetime
 
 from Cryptodome.Cipher import AES
 
@@ -87,39 +88,30 @@ def get_hotspot_id(self, name_str, lat, lng):
             return cur.lastrowid
 
     # Insert an entry into the hotspot connections table.
-    def record_hotspot(self, report_id, rec):
+    def record_hotspot(self, report_id, rec, frequency_hZ):
         sql = 'INSERT INTO hotspot_connections (report_id, frequency, name_id, rssi, snr) VALUES (?, ?, ?, ?, ?)'
         vals = (report_id,
-                float(rec['frequency']),
-                self.get_hotspot_id(rec['name'], float(rec['lat']), float(rec['long'])),
+                int(frequency_hZ),
+                self.get_hotspot_id(rec['metadata']['gateway_name'], float(rec['metadata']['gateway_lat']), float(rec['metadata']['gateway_long'])),
                 float(rec['rssi']),
                 float(rec['snr']))
         cur = self.conn.cursor()
         cur.execute(sql, vals)
         self.conn.commit()
 
-    # Insert the device label (name), as described in the Helium integration
-    # and passed in the Json metadata.
-    def record_label(self, report_id, j):
-        sql = 'INSERT INTO label_reports (report_id, name_id) VALUES (?, ?)'
-        vals = (report_id,
-                self.get_id_from_string('label_strings', j['name']))
-        cur = self.conn.cursor()
-        cur.execute(sql, vals)
-        self.conn.commit()
-
     # Insert a new report entry.
     def record_report(self, rec, battery_voltage):
-        sql = 'INSERT INTO reports (app_eui_id, dev_eui_id, dev_addr_id, dc_balance, fcnt, port, name_id, battery_voltage, reported_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
-        vals = (self.get_id_from_string('app_eui', rec['app_eui']),
-                self.get_id_from_string('dev_eui', rec['dev_eui']),
-                self.get_id_from_string('dev_addr', rec['devaddr']),
-                int(rec['dc']['balance']), 
-                int(rec['fcnt']), 
-                int(rec['port']), 
-                self.get_id_from_string('device_names', rec['name']),
+        sql = 'INSERT INTO reports (dev_eui_id, dev_addr_id, dc_balance, fcnt, port, name_id, profile_id, battery_voltage, reported_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
+        epoch_timestamp_ms = datetime.datetime.fromisoformat(rec['time']).timestamp() * 1000
+        vals = (self.get_id_from_string('dev_eui', rec['deviceInfo']['devEui']),
+                self.get_id_from_string('dev_addr', rec['devAddr']),
+                int(rec['dc']['balance'] if 'dc' in rec else -1),
+                int(rec['fCnt']),
+                int(rec['fPort']),
+                self.get_id_from_string('device_names', rec['deviceInfo']['deviceName']),
+                self.get_id_from_string('profile_names', rec['deviceInfo']['deviceProfileName']),
                 battery_voltage,
-                int(rec['reported_at']))
+                int(epoch_timestamp_ms))
         cur = self.conn.cursor()
         cur.execute(sql, vals)
         self.conn.commit()
@@ -143,21 +135,15 @@ def record_measurement(self, report_id, payload):
     def record(self, json_str):
         rec = json.loads(json_str)
 
-        if rec['type'] == 'join':
-            print('Received a JOIN packet.')
-            return
-
         payload = Payload()
-        payload.decode(rec['payload'])
+        payload.decode(rec['data'])
         print(f'T={payload.temperature}°C, P={payload.pressure_Pa/100}hPa, RH={payload.humidity_RH}%, BAT={payload.battery_voltage}mV')
 
         report_id = self.record_report(rec, payload.battery_voltage)
 
         self.record_measurement(report_id, payload)
-        for hotspot in rec['hotspots']:
-            self.record_hotspot(report_id, hotspot)
-        for label in rec['metadata']['labels']:
-            self.record_label(report_id, label)
+        for hotspot in rec['rxInfo']:
+            self.record_hotspot(report_id, hotspot, float(rec['txInfo']['frequency']))
 
 if __name__ == '__main__':
     t = Meteo()

integration/queries/dump-data.sql

@@ -1,6 +1,5 @@
-SELECT reports.reported_at_ms, reports.dc_balance, COUNT(hotspot_connections.id), reports.fcnt, device_names.name, reports.battery_voltage, measurements.temperature, measurements.pressure, measurements.humidity
-FROM ((((((reports
-INNER JOIN app_eui ON app_eui.id = reports.app_eui_id)
+SELECT datetime(reports.reported_at_ms / 1000, 'unixepoch', 'localtime'), reports.dc_balance, COUNT(hotspot_connections.id), reports.fcnt, device_names.name, reports.battery_voltage, measurements.temperature, measurements.pressure, measurements.humidity
+FROM (((((reports
 INNER JOIN dev_eui ON dev_eui.id = reports.dev_eui_id)
 INNER JOIN dev_addr ON dev_addr.id = reports.dev_addr_id)
 INNER JOIN device_names ON device_names.id = reports.name_id)

integration/server.py

@@ -27,20 +27,33 @@ def do_GET(self):
         self.wfile.write("42".encode('utf-8'))
 
     def do_POST(self):
+        self.send_response(200)
+        self.end_headers()
+
         content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
         post_data = self.rfile.read(content_length) # <--- Gets the data itself
         logging.debug("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",
                 str(self.path), str(self.headers), post_data.decode('utf-8'))
         try:
-           self.meteo.record(post_data)
+            event = 'unknown'
+            for arg in self.path.split('?'):
+                arg_split = arg.split('=', 3)
+                if len(arg_split) != 2:
+                    continue
+                path_key, path_val = arg_split
+                if path_key == 'event':
+                    event = path_val
+                    break
+            if event == 'up':
+                self.meteo.record(post_data)
+            else:
+                print('Ignoring event ' + event)
         except Exception as e:
             print('Exception occurred with the following json: {}'.format(post_data))
             print('Exception: ' + str(e))
         logging.info('Received: json: {}'.format(post_data))
-        self._set_response()
-        self.wfile.write("42".encode('utf-8'))
 
-def run(server_class=HTTPServer, handler_class=Server, port=8082):
+def run(server_class=HTTPServer, handler_class=Server, port=8085):
     logging.basicConfig(level=logging.INFO)
     server_address = ('', port)
     httpd = server_class(server_address, handler_class)