feat: Handle the case when we get gas data from the past by ignoring …

code_style.sh

@@ -2,4 +2,6 @@
 
 set -e
 
+mypy bin/glowprom glowprom/ tests/
+
 black .

glowprom/local_message.py

@@ -16,6 +16,8 @@
 
 import datetime
 import json
+from typing import Any, Dict, Tuple
+import logging
 
 # {'electricitymeter': {'timestamp': '2022-11-07T09:20:08Z',
 #   'energy': {'export': {'cumulative': 0.0, 'units': 'kWh'},
@@ -66,15 +68,15 @@
 METRIC_HELP = "# HELP {metric} {help}"
 METRIC_TYPE = "# TYPE {metric} {type}"
 
-ELECTRIC_DATA = {}
-GAS_DATA = {}
-STATE_TIMESTAMP = None
+ELECTRIC_DATA: Dict[str, Dict[str, float | Tuple[float, str]]] = {}
+GAS_DATA: Dict[str, Dict[str, float | Tuple[float, str]]] = {}
+STATE_TIMESTAMP: float | None = None
 VERSIONS = {}
 RSSI = None
 LQI = None
 
 
-def local_message(msg):
+def local_message(msg: Any) -> str:
     # # Code adapted from
     # # https://gist.github.com/ndfred/b373eeafc4f5b0870c1b8857041289a9
     payload = json.loads(msg.payload)
@@ -84,7 +86,7 @@ def local_message(msg):
 
         mpan = energy["import"]["mpan"]
         if mpan.lower() == "read pending":  # pragma: no cover
-            return
+            return generate_metrics()
         if mpan not in ELECTRIC_DATA:
             ELECTRIC_DATA[mpan] = {}
         ELECTRIC_DATA[mpan]["timestamp"] = datetime.datetime.strptime(
@@ -118,12 +120,25 @@ def local_message(msg):
         energy = payload["gasmeter"]["energy"]
         mprn = energy["import"]["mprn"]
         if mprn.lower() == "read pending":  # pragma: no cover
-            return
+            return generate_metrics()
         if mprn not in GAS_DATA:
             GAS_DATA[mprn] = {}
-        GAS_DATA[mprn]["timestamp"] = datetime.datetime.strptime(
+
+        timestamp = datetime.datetime.strptime(
             payload["gasmeter"]["timestamp"], r"%Y-%m-%dT%H:%M:%SZ"
         ).timestamp()
+
+        if "timestamp" in GAS_DATA[mprn]:
+            previous_timestamp = GAS_DATA[mprn]["timestamp"]
+            assert isinstance(previous_timestamp, float)
+            if timestamp < previous_timestamp:
+                logging.warning(
+                    f"Ignoring gas message, time has gone backwards - {timestamp} < {GAS_DATA[mprn]["timestamp"]}"
+                )
+                logging.warning(f"Dropping {payload["gasmeter"]}")
+                return generate_metrics()
+
+        GAS_DATA[mprn]["timestamp"] = timestamp
         GAS_DATA[mprn]["import_cumulative"] = convert_units(
             energy["import"]["cumulative"], energy["import"]["units"]
         )
@@ -162,8 +177,12 @@ def local_message(msg):
         RSSI = payload["han"]["rssi"]
         LQI = payload["han"]["lqi"]
     else:  # pragma: no cover
-        print(f"Unknown payload type {key}")
+        print(f"Unknown payload type {payload.keys()}")
+
+    return generate_metrics()
+
 
+def generate_metrics() -> str:
     lines = []
     for metric in METRIC_METADATA.keys():
         help, metric_type, has_units = METRIC_METADATA[metric]
@@ -180,29 +199,29 @@ def local_message(msg):
 
         for mpan in ELECTRIC_DATA:
             if metric in ELECTRIC_DATA[mpan]:
+                metric_value = ELECTRIC_DATA[mpan][metric]
                 if has_units:
-                    value = ELECTRIC_DATA[mpan][metric][0]
-                    metric_name = (
-                        METRIC.format(metric=metric)
-                        + "_"
-                        + ELECTRIC_DATA[mpan][metric][1]
-                    )
+                    assert isinstance(metric_value, tuple)
+                    value = metric_value[0]
+                    metric_name = METRIC.format(metric=metric) + "_" + metric_value[1]
                 else:
-                    value = ELECTRIC_DATA[mpan][metric]
+                    assert isinstance(metric_value, float)
+                    value = metric_value
                     metric_name = METRIC.format(metric=metric)
 
                 keys = METRIC_KEYS.format(type="electric", idname="mpan", idvalue=mpan)
                 lines.append(f"{metric_name}{keys} {value}")
 
         for mprn in GAS_DATA:
             if metric in GAS_DATA[mprn]:
+                metric_value = GAS_DATA[mprn][metric]
                 if has_units:
-                    value = GAS_DATA[mprn][metric][0]
-                    metric_name = (
-                        METRIC.format(metric=metric) + "_" + GAS_DATA[mprn][metric][1]
-                    )
+                    assert isinstance(metric_value, tuple)
+                    value = metric_value[0]
+                    metric_name = METRIC.format(metric=metric) + "_" + metric_value[1]
                 else:
-                    value = GAS_DATA[mprn][metric]
+                    assert isinstance(metric_value, float)
+                    value = metric_value
                     metric_name = METRIC.format(metric=metric)
 
                 keys = METRIC_KEYS.format(type="gas", idname="mprn", idvalue=mprn)

glowprom/server.py

@@ -35,14 +35,18 @@ def do_GET(self):
     def send_index(self):
         self.send_response(200)
         self.end_headers()
-        self.wfile.write("""
+        self.wfile.write(
+            """
 <html>
 <head><title>Glow Prometheus</title></head>
 <body>
 <h1>Glow Prometheus</h1>
 <p><a href="/metrics">Metrics</a></p>
 </body>
-</html>""".encode("utf8"))
+</html>""".encode(
+                "utf8"
+            )
+        )
 
     def send_metrics(self):
         if STATS is None:

requirements.txt

@@ -6,3 +6,4 @@ packaging==26.0
 wheel==0.46.3
 twine==6.2.0
 setuptools==82.0.1
+mypy==1.19.1

tests/test_arguments.py

@@ -35,7 +35,7 @@ def test_passwd_environ(self):
         args = get_arguments(["--user", "testuser", "--topic", "topic"])
         self.assertEqual("testpassword", args.passwd)
 
-    def test_passwd_environ(self):
+    def test_topic_environ(self):
         os.environ["GLOWPROM_TOPIC"] = "topic"
         args = get_arguments(["--user", "testuser", "--passwd", "testpassword"])
         self.assertEqual("topic", args.topic)

tests/test_local_gas_backwards_message.txt

@@ -0,0 +1,12 @@
+{"gasmeter": {"timestamp": "2021-11-07T09:35:38Z",
+   "energy": {"import": {"cumulative": 0, "day": 10,
+                         "week": 0, "month": 0, "units": "kWh",
+                         "cumulativevol": 0,
+                         "cumulativevolunits": "m3",
+                         "dayvol": 0, "weekvol": 0,
+                         "monthvol": 0,
+                         "dayweekmonthvolunits": "kWh",
+                         "mprn": "wxyz",
+                         "supplier": "---",
+                         "price": {"unitrate": 0.03623,
+                                   "standingcharge": 0.168}}}}}

tests/test_local_message.py

@@ -21,6 +21,9 @@
 
 ELECTRIC_MESSAGE_TEXT = open("tests/test_local_electric_message.txt", "rb").read()
 GAS_MESSAGE_TEXT = open("tests/test_local_gas_message.txt", "rb").read()
+GAS_MESSAGE_BACKWARDS_TEXT = open(
+    "tests/test_local_gas_backwards_message.txt", "rb"
+).read()
 STATE_MESSAGE_TEXT = open("tests/test_local_state_message.txt", "rb").read()
 
 
@@ -48,6 +51,23 @@ def test_gas_message(self):
             prom,
         )
 
+    def test_gas_message_ignore_backwards(self):
+        prom = local_message(MockMessage(GAS_MESSAGE_TEXT))
+
+        self.assertIn(
+            'glowprom_import_cumulative_Wh{type="gas", mprn="wxyz"}'
+            + " 66589570.00000001",
+            prom,
+        )
+
+        prom = local_message(MockMessage(GAS_MESSAGE_BACKWARDS_TEXT))
+
+        self.assertIn(
+            'glowprom_import_cumulative_Wh{type="gas", mprn="wxyz"}'
+            + " 66589570.00000001",
+            prom,
+        )
+
     def test_state_message(self):
         prom = local_message(MockMessage(STATE_MESSAGE_TEXT))