refactor(parser): remove arrow in a few straggler files (#7636)

parsers/BORNHOLM_POWERLAB.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# The arrow library is used to handle datetimes
 from datetime import datetime
 from logging import Logger, getLogger
 from zoneinfo import ZoneInfo

parsers/IN_DL.py

@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
 
-from datetime import datetime
+from datetime import datetime, time
 from logging import Logger, getLogger
+from zoneinfo import ZoneInfo
 
 from requests import Session
 
 from .lib import IN, web, zonekey
 
+ZONE_INFO = ZoneInfo
+
 plants = {
     "CCGT-Bawana": "Gas",
     "DMSWSL-Dsidc": "G2E",
@@ -34,14 +37,14 @@ def fetch_consumption(
     )
 
     india_date_time = IN.read_datetime_from_span_id(
-        html, "DynamicData1_LblDate", "DD-MMM-YYYY hh:mm:ss A"
+        html, "DynamicData1_LblDate", "%d-%b-%Y %I:%M:%S %p"
     )
 
     demand_value = IN.read_value_from_span_id(html, "DynamicData1_LblLoad")
 
     data = {
         "zoneKey": zone_key,
-        "datetime": india_date_time.datetime,
+        "datetime": india_date_time,
         "consumption": demand_value,
         "source": "delhisldc.org",
     }
@@ -67,8 +70,11 @@ def fetch_production(
         zone_key, "http://www.delhisldc.org/Redirect.aspx?Loc=0804", session
     )
 
-    india_date_string = IN.read_text_from_span_id(html, "ContentPlaceHolder3_ddgenco")
-    india_date_time = IN.read_datetime_with_only_time(india_date_string, "HH:mm:ss")
+    india_time_string = IN.read_text_from_span_id(html, "ContentPlaceHolder3_ddgenco")
+    india_time = time.fromisoformat(india_time_string)
+    india_date_time = datetime.combine(datetime.now(IN.ZONE_INFO), india_time).replace(
+        tzinfo=IN.ZONE_INFO
+    )
 
     prod_table = html.find("table", {"id": "ContentPlaceHolder3_dgenco"})
     prod_rows = prod_table.findAll("tr")
@@ -78,7 +84,7 @@ def fetch_production(
 
     data = {
         "zoneKey": zone_key,
-        "datetime": india_date_time.datetime,
+        "datetime": india_date_time,
         "production": {
             "coal": energy["Coal"],
             "gas": energy["Gas"],

parsers/IN_KA.py

@@ -22,13 +22,13 @@ def fetch_consumption(
     zonekey.assert_zone_key(zone_key, "IN-KA")
     html = web.get_response_soup(zone_key, "http://kptclsldc.in/Default.aspx", session)
 
-    india_date_time = IN.read_datetime_from_span_id(html, "Label6", "DD/MM/YYYY HH:mm")
+    india_date_time = IN.read_datetime_from_span_id(html, "Label6", "%d/%m/%Y %H:%M")
 
     demand_value = IN.read_value_from_span_id(html, "Label5")
 
     data = {
         "zoneKey": zone_key,
-        "datetime": india_date_time.datetime,
+        "datetime": india_date_time,
         "consumption": demand_value,
         "source": "kptclsldc.in",
     }
@@ -51,7 +51,7 @@ def fetch_production(
     html = web.get_response_soup(zone_key, "http://kptclsldc.in/StateGen.aspx", session)
 
     india_date_time = IN.read_datetime_from_span_id(
-        html, "lbldate", "DD/MM/YYYY HH:mm:ss"
+        html, "lbldate", "%d/%m/%Y %H:%M:%S"
     )
 
     # RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station
@@ -129,7 +129,7 @@ def fetch_production(
         zone_key, "http://kptclsldc.in/StateNCEP.aspx", session
     )
     ncep_date_time = IN.read_datetime_from_span_id(
-        ncep_html, "Label1", "DD/MM/YYYY HH:mm:ss"
+        ncep_html, "Label1", "%d/%m/%Y %H:%M:%S"
     )
 
     # Check ncep date is similar than state gen date
@@ -176,7 +176,7 @@ def fetch_production(
 
     data = {
         "zoneKey": zone_key,
-        "datetime": india_date_time.datetime,
+        "datetime": india_date_time,
         "production": {
             "biomass": biomass_value,
             "coal": coal_value,

parsers/archived/IN_AP.py

@@ -7,7 +7,7 @@
 
 URL = "https://core.ap.gov.in/CMDashBoard/UserInterface/Power/PowerReport.aspx"
 ZONE_KEY = "IN-AP"
-TIME_FORMAT = "DD-MM-YYYY HH:mm"
+TIME_FORMAT = "%d-%m-%Y %H:%M"
 SOURCE = "core.ap.gov.in"
 
 
@@ -42,7 +42,7 @@ def fetch_production(
 
     return {
         "zoneKey": zone_key,
-        "datetime": india_date.datetime,
+        "datetime": india_date,
         "production": {
             "biomass": 0.0,
             "coal": thermal_value,
@@ -81,7 +81,7 @@ def fetch_consumption(
 
     return {
         "zoneKey": zone_key,
-        "datetime": india_date.datetime,
+        "datetime": india_date,
         "consumption": demand_value,
         "source": SOURCE,
     }

parsers/lib/IN.py

@@ -1,12 +1,12 @@
-from arrow import get, utcnow
+from datetime import datetime
+from zoneinfo import ZoneInfo
 
-TZ = "Asia/Kolkata"
+ZONE_INFO = ZoneInfo("Asia/Kolkata")
 
 
 def read_datetime_from_span_id(html, span_id, time_format):
     date_time_span = html.find("span", {"id": span_id})
-    india_date_time = date_time_span.text + " Asia/Kolkata"
-    return get(india_date_time, time_format + " ZZZ")
+    return datetime.strptime(date_time_span.text, time_format).replace(tzinfo=ZONE_INFO)
 
 
 def read_text_from_span_id(html, span_id):
@@ -16,15 +16,3 @@ def read_text_from_span_id(html, span_id):
 def read_value_from_span_id(html, span_id):
     html_span = read_text_from_span_id(html, span_id)
     return float(html_span)
-
-
-def read_datetime_with_only_time(time_string, time_format, now=utcnow()):
-    utc = now.floor("hour")
-    india_now = utc.to(TZ)
-    time = get(time_string, time_format)
-    india_date_time = india_now.replace(
-        hour=time.hour, minute=time.minute, second=time.second
-    )
-    if india_date_time > india_now:
-        india_date_time.shift(days=-1)
-    return india_date_time

parsers/lib/quality.py

@@ -7,8 +7,6 @@
 from typing import Any
 from warnings import warn
 
-import arrow
-
 from electricitymap.contrib.config import EXCHANGES_CONFIG, emission_factors
 from electricitymap.contrib.lib.types import ZoneKey
 
@@ -40,16 +38,16 @@ def validate_datapoint_format(datapoint: dict[str, Any], kind: str, zone_key: Zo
 
 
 def validate_reasonable_time(item, k):
-    data_time = arrow.get(item["datetime"])
-    if data_time.year < 2000:
+    data_dt = item["datetime"].astimezone(timezone.utc)
+    now = datetime.now(timezone.utc)
+
+    if data_dt.year < 2000:
         raise ValidationError(
-            f"Data from {k} can't be before year 2000, it was from: {data_time}"
+            f"Data from {k} can't be before year 2000, it was from: {data_dt}"
         )
-
-    arrow_now = arrow.utcnow()
-    if data_time.astimezone(timezone.utc) > arrow_now:
+    if data_dt > now:
         raise ValidationError(
-            f"Data from {k} can't be in the future, data was {data_time}, now is {arrow_now}"
+            f"Data from {k} can't be in the future, it was from {data_dt}, now is {now}"
         )
 
 

parsers/test/lib/test_IN.py

@@ -1,6 +1,5 @@
 import unittest
 
-from arrow import get
 from bs4 import BeautifulSoup
 
 from parsers.lib import IN
@@ -11,15 +10,15 @@ def test_read_datetime_from_span_id(self):
         html_span = '<p><span id="lbldate">9/4/2017 5:17:00 PM</span></p>'
         html = BeautifulSoup(html_span, "html.parser")
         india_date_time = IN.read_datetime_from_span_id(
-            html, "lbldate", "D/M/YYYY h:mm:ss A"
+            html, "lbldate", "%d/%m/%Y %I:%M:%S %p"
         )
         self.assertIsNotNone(india_date_time)
         self.assertEqual(india_date_time.isoformat(), "2017-04-09T17:17:00+05:30")
 
         html_span = '<p><span id="lblPowerStatusDate">04-09-2017 17:13</span></p>'
         html = BeautifulSoup(html_span, "html.parser")
         india_date_time = IN.read_datetime_from_span_id(
-            html, "lblPowerStatusDate", "DD-MM-YYYY HH:mm"
+            html, "lblPowerStatusDate", "%d-%m-%Y %H:%M"
         )
         self.assertIsNotNone(india_date_time)
         self.assertEqual(india_date_time.isoformat(), "2017-09-04T17:13:00+05:30")
@@ -38,25 +37,6 @@ def test_read_value_from_span_id(self):
         self.assertIsNotNone(cgs_value)
         self.assertEqual(cgs_value, 2998.0)
 
-    def test_read_india_datetime_with_only_time(self):
-        mock_now = get("2017-11-01T19:05:01+00:00")
-        time_string = "01:05:01"
-        time_format = "HH:mm:ss"
-        india_date_time = IN.read_datetime_with_only_time(
-            time_string, time_format, mock_now
-        )
-        self.assertIsNotNone(india_date_time)
-        self.assertEqual(india_date_time.isoformat(), "2017-11-02T01:05:01+05:30")
-
-        mock_now = get("2017-11-02T01:05:01+00:00")
-        time_string = "06:35:01"
-        time_format = "HH:mm:ss"
-        india_date_time = IN.read_datetime_with_only_time(
-            time_string, time_format, mock_now
-        )
-        self.assertIsNotNone(india_date_time)
-        self.assertEqual(india_date_time.isoformat(), "2017-11-02T06:35:01+05:30")
-
 
 if __name__ == "__main__":
     unittest.main()

parsers/test/test_IN.py

@@ -1,21 +1,26 @@
 import unittest
-
-import arrow
+from datetime import datetime
 
 from parsers.IN import get_start_of_day
 
 
 class TestIN(unittest.TestCase):
     def test_get_start_of_day(self):
         # "2020-12-17 00:00:00+05:30" -> "2020-12-16 18:30:00+00:00"
-        start_day = get_start_of_day(arrow.get("2020-12-16 18:30:00+00:00").datetime)
-        assert start_day == arrow.get("2020-12-17 00:00:00+05:30").datetime
+        start_day = get_start_of_day(
+            datetime.fromisoformat("2020-12-16 18:30:00+00:00")
+        )
+        assert start_day == datetime.fromisoformat("2020-12-17 00:00:00+05:30")
         # "2020-12-16 23:59:00+05:30" -> "2020-12-16 18:29:00+00:00"
-        start_day = get_start_of_day(arrow.get("2020-12-16 18:29:00+00:00").datetime)
-        assert start_day == arrow.get("2020-12-16 00:00:00+05:30").datetime
+        start_day = get_start_of_day(
+            datetime.fromisoformat("2020-12-16 18:29:00+00:00")
+        )
+        assert start_day == datetime.fromisoformat("2020-12-16 00:00:00+05:30")
         # "2020-12-17 00:05:30+05:30" -> "2020-12-17 00:00:00+00:00"
-        start_day = get_start_of_day(arrow.get("2020-12-17 00:00:00+00:00").datetime)
-        assert start_day == arrow.get("2020-12-17 00:00:00+05:30").datetime
+        start_day = get_start_of_day(
+            datetime.fromisoformat("2020-12-17 00:00:00+00:00")
+        )
+        assert start_day == datetime.fromisoformat("2020-12-17 00:00:00+05:30")
 
 
 if __name__ == "__main__":

parsers/test/test_OPENNEM.py

@@ -1,6 +1,6 @@
 import unittest
+from datetime import datetime, timedelta, timezone
 
-import arrow
 import numpy as np
 import pandas as pd
 
@@ -66,10 +66,10 @@ def test_sum_vector(self):
         assert sum_wind == sum(values_wind)
 
     def test_filter_production_objs(self):
-        now = arrow.utcnow()
+        now = datetime.now(timezone.utc)
         objs = [
             {
-                "datetime": now.shift(hours=-1).datetime,
+                "datetime": now - timedelta(hours=1),
                 "production": {
                     "coal": 12,
                     "solar": 1.0,
@@ -79,7 +79,7 @@ def test_filter_production_objs(self):
                 },
             },
             {
-                "datetime": now.shift(hours=-2).datetime,
+                "datetime": now - timedelta(hours=2),
                 "production": {
                     "coal": 12,
                     "solar": None,