36 add function for number of measurements per yearday (#63)

* added ddlpy.measurements_amount() function

* added testcase

* updated history.rst

* removed inbetween python version from pytest

.github/workflows/pytest.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         # we assume it also works for inbetween versions
-        python-version: ["3.8", "3.11", "3.12"]
+        python-version: ["3.8", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
 

HISTORY.rst

@@ -13,6 +13,7 @@ UNRELEASED
 * consistency improvements for `ddlpy.measurements()` output dataframe in https://github.com/deltares/ddlpy/pull/45
 * add distinction for Groepering (timeseries vs. extremes) to `ddlpy.locations()` dataframe in https://github.com/deltares/ddlpy/pull/49
 * drop `Tijdstip` column in `ddlpy.measurements()` output dataframe to avoid duplication with time index in https://github.com/deltares/ddlpy/pull/52 and https://github.com/deltares/ddlpy/pull/54
+* add `ddlpy.measurements_amount()` to retrieve the number of available measurements grouped by day/month/year in https://github.com/Deltares/ddlpy/pull/63
 
 0.1.0 (2019-01-03)
 ------------------

ddlpy/__init__.py

@@ -10,12 +10,14 @@
 from ddlpy.ddlpy import (measurements, 
                          measurements_latest, 
                          measurements_available, 
+                         measurements_amount,
                          simplify_dataframe,
                          )
 
 __all__ = ['locations', 
            'measurements',
            'measurements_latest', 
            'measurements_available', 
+           'measurements_amount',
            'simplify_dataframe',
            ]

ddlpy/ddlpy.py

@@ -129,6 +129,68 @@ def measurements_available(location, start_date, end_date):
             return False  
 
 
+def measurements_amount(location, start_date, end_date, period="Jaar"):
+    """checks how much measurements are available for a location, for the period start_date, end_date
+    returns a DataFrame with columns Groeperingsperiode and AantalMetingen
+    possible for Jaar/Maand/Dag
+    """
+    # TODO: there are probably more Groeperingsperiodes accepted by ddl, but not supported by ddlpy yet
+    accepted_period = ["Jaar","Maand","Dag"]
+    if period not in accepted_period:
+        raise ValueError(f"period should be one of {accepted_period}, not '{period}'")
+
+    endpoint = ENDPOINTS['collect_number_of_observations']
+
+    start_date = pd.Timestamp(start_date)
+    end_date = pd.Timestamp(end_date)
+
+    start_date_str = pytz.UTC.localize(start_date).isoformat(timespec='milliseconds')
+    end_date_str = pytz.UTC.localize(end_date).isoformat(timespec='milliseconds')
+
+    request_dicts = _get_request_dicts(location)
+
+    request = {
+        "AquoMetadataLijst": [request_dicts["AquoMetadata"]],
+        "LocatieLijst": [request_dicts["Locatie"]],
+        "Groeperingsperiode": period,
+        "Periode": {
+            "Begindatumtijd": start_date_str,
+            "Einddatumtijd": end_date_str
+        }
+    }
+
+    try:
+        logger.debug('requesting:  {}'.format(request))
+        resp = requests.post(endpoint['url'], json=request)
+        result = resp.json()
+        if not result['Succesvol']:
+            logger.debug('Got  invalid response: {}'.format(result))
+            raise NoDataException(result.get('Foutmelding', 'No error returned'))
+    except NoDataException as e:
+        logger.debug('No data availble for {} {}'.format(start_date, end_date))
+        raise e
+
+    if result['Succesvol']:
+        df_list = []
+        for one in result['AantalWaarnemingenPerPeriodeLijst']:
+            df = pd.json_normalize(one['AantalMetingenPerPeriodeLijst'])
+
+            # combine columns to a period string
+            df["Groeperingsperiode"] = df["Groeperingsperiode.Jaarnummer"].apply(lambda x: f"{x:04d}")
+            if period in ["Maand", "Dag"]:
+                df["Groeperingsperiode"] = df["Groeperingsperiode"] + "-" + df["Groeperingsperiode.Maandnummer"].apply(lambda x: f"{x:02d}")
+            if period in ["Dag"]:
+                df["Groeperingsperiode"] = df["Groeperingsperiode"] + "-" + df["Groeperingsperiode.Dag"].apply(lambda x: f"{x:02d}")
+
+            # select columns from dataframe and append to list
+            df = df[["Groeperingsperiode","AantalMetingen"]]
+            df_list.append(df)
+
+        # concatenate
+        amount_all = pd.concat(df_list).sort_values("Groeperingsperiode")
+        return amount_all
+
+
 def _combine_waarnemingenlijst(result, location):
     assert "WaarnemingenLijst" in result
 

tests/test_ddlpy.py

@@ -32,6 +32,20 @@ def test_measurements_available(location):
     assert isinstance(data_present, bool)
 
 
+def test_measurements_amount(location):
+    start_date = dt.datetime(1953, 1, 1)
+    end_date = dt.datetime(1953, 4, 5)
+    data_amount_dag = ddlpy.measurements_amount(location, start_date=start_date, end_date=end_date, period="Dag")
+    assert data_amount_dag.shape[0] > 50
+    assert data_amount_dag["Groeperingsperiode"].str.len().iloc[0] == 10
+    data_amount_maand = ddlpy.measurements_amount(location, start_date=start_date, end_date=end_date, period="Maand")
+    assert data_amount_maand.shape[0] == 4
+    assert data_amount_maand["Groeperingsperiode"].str.len().iloc[0] == 7
+    data_amount_jaar = ddlpy.measurements_amount(location, start_date=start_date, end_date=end_date, period="Jaar")
+    assert data_amount_jaar.shape[0] == 1
+    assert data_amount_jaar["Groeperingsperiode"].str.len().iloc[0] == 4
+
+
 def test_measurements(location):
     """measurements for a location """
     start_date = dt.datetime(1953, 1, 1)