Merge branch 'main' into 85-create-to_xarray-function

HISTORY.rst

@@ -4,7 +4,8 @@ History
 
 UNRELEASED
 ------------------
-* nothing yet
+* added `catalog_filter` argument to `ddlpy.locations()` to enabling retrieving the extended catalog in https://github.com/Deltares/ddlpy/pull/87
+* pass all Code parameters to measurements request instead of only four in https://github.com/Deltares/ddlpy/pull/88
 
 0.3.0 (2023-03-13)
 ------------------

ddlpy/ddlpy.py

@@ -30,24 +30,38 @@ class NoDataException(ValueError):
 logger = logging.getLogger(__name__)
 
 
-def locations():
-    """
-    get station information from DDL (metadata from Catalogue). All metadata regarding stations.
-    The response (result) retrieves more keys
-
-    """
+def catalog(catalog_filter=None):
     endpoint = ENDPOINTS["collect_catalogue"]
-    msg = "{} with {}".format(endpoint["url"], json.dumps(endpoint["request"]))
+
+    if catalog_filter is None:
+        # use the default request from endpoints.json
+        catalog_request = endpoint["request"]
+    else:
+        assert isinstance(catalog_filter, list)
+        catalog_request = {"CatalogusFilter": {x:True for x in catalog_filter}}
+
+    msg = "{} with {}".format(endpoint["url"], json.dumps(catalog_request))
     logger.debug("requesting: {}".format(msg))
 
-    resp = requests.post(endpoint["url"], json=endpoint["request"])
+    resp = requests.post(endpoint["url"], json=catalog_request)
     if not resp.ok:
         raise IOError("Failed to request {}: {}".format(msg, resp.text))
     result = resp.json()
     if not result["Succesvol"]:
         logger.exception(str(result))
         raise ValueError(result.get("Foutmelding", "No error returned"))
+    return result
+
 
+def locations(catalog_filter=None):
+    """
+    get station information from DDL (metadata from Catalogue). All metadata regarding stations.
+    The response (result) retrieves more keys
+
+    """
+
+    result = catalog(catalog_filter=catalog_filter)
+
     df_locations = pd.DataFrame(result["LocatieLijst"])
 
     df_metadata = pd.json_normalize(result["AquoMetadataLijst"])
@@ -82,13 +96,12 @@ def _check_convert_dates(start_date, end_date, return_str=True):
 
 
 def _get_request_dicts(location):
-    aquometadata_dict = {
-        "Eenheid": {"Code": location["Eenheid.Code"]},
-        "Grootheid": {"Code": location["Grootheid.Code"]},
-        "Hoedanigheid": {"Code": location["Hoedanigheid.Code"]},
-        "Groepering": {"Code": location["Groepering.Code"]},
-    }
 
+    # generate aquometadata dict from location "*.Code" values
+    key_list = [x.replace(".Code","") for x in location.index if x.endswith(".Code")]
+    aquometadata_dict = {key:{"Code":location[f"{key}.Code"]} for key in key_list}
+
+    # generate location dict from relevant values
     locatie_dict = {
         "X": location["X"],
         "Y": location["Y"],
@@ -203,7 +216,6 @@ def measurements_amount(location, start_date, end_date, period="Jaar"):
 def _combine_waarnemingenlijst(result, location):
     assert "WaarnemingenLijst" in result
 
-    # assert len(result['WaarnemingenLijst']) == 1
     # flatten the datastructure
     rows = []
     for waarneming in result["WaarnemingenLijst"]:

tests/test_ddlpy.py

@@ -34,9 +34,23 @@ def measurements(location):
 
 
 def test_locations(locations):
+    # the number of columns depend on the catalog filter in endpoints.json
+    assert locations.shape[1] == 18
+    # the number of rows is the number of stations, so will change over time
     assert locations.shape[0] > 1
 
 
+def test_locations_extended():
+    catalog_filter = ['Compartimenten','Eenheden','Grootheden',
+                      'Hoedanigheden','Groeperingen','MeetApparaten',
+                      'Typeringen','WaardeBepalingsmethoden','Parameters']
+    locations_extended = ddlpy.locations(catalog_filter=catalog_filter)
+    # the number of columns depend on the provided catalog_filter
+    assert locations_extended.shape[1] == 24
+    # the number of rows is the number of stations, so will change over time
+    assert locations_extended.shape[0] > 1
+
+
 def test_measurements(measurements):
     assert measurements.shape[0] > 1