Add get_solargis iotools function (#1969)

AdamRJensen · web-flow · commit 3c7b5dae782a · 2024-03-11T13:58:42.000-04:00
* Initial commit

* Linting

* Include v0.10.4 whatsnew

* Add example

* Update raises message

* Update doc section order

* Set parser to etree

* Add tests

* Add test skip for pandas&lt;1.3.0

* Fix nan replacement and add variable conversion

* Add time_resolution conversion

* Fix precipitable_water conversion in solcast

* Address reviewer comments

* Fix test issues

* Update pandas version in ci/requirements*.yml

* Update test_solcast.py

* Advance numpy requirement
diff --git a/ci/requirements-py3.10.yml b/ci/requirements-py3.10.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/ci/requirements-py3.11.yml b/ci/requirements-py3.11.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/ci/requirements-py3.12.yml b/ci/requirements-py3.12.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/ci/requirements-py3.7-min.yml b/ci/requirements-py3.7-min.yml
@@ -14,8 +14,8 @@ dependencies:
     - pip:
         - dataclasses
         - h5py==3.1.0
-        - numpy==1.16.0
-        - pandas==0.25.0
+        - numpy==1.17.3
+        - pandas==1.3.0
         - scipy==1.5.0
         - pytest-rerunfailures # conda version is >3.6
         - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1
diff --git a/ci/requirements-py3.7.yml b/ci/requirements-py3.7.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/ci/requirements-py3.8.yml b/ci/requirements-py3.8.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/ci/requirements-py3.9.yml b/ci/requirements-py3.9.yml
@@ -8,8 +8,8 @@ dependencies:
     - ephem
     - h5py
     - numba
-    - numpy >= 1.16.0
-    - pandas >= 0.25.0
+    - numpy >= 1.17.3
+    - pandas >= 1.3.0
     - pip
     - pytest
     - pytest-cov
diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst
@@ -53,6 +53,7 @@ of sources and file formats relevant to solar energy modeling.
    iotools.get_solcast_historic
    iotools.get_solcast_forecast
    iotools.get_solcast_live
+   iotools.get_solargis
 
 
 A :py:class:`~pvlib.location.Location` object may be created from metadata
diff --git a/docs/sphinx/source/whatsnew.rst b/docs/sphinx/source/whatsnew.rst
@@ -6,6 +6,7 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.10.4.rst
 .. include:: whatsnew/v0.10.3.rst
 .. include:: whatsnew/v0.10.2.rst
 .. include:: whatsnew/v0.10.1.rst
diff --git a/docs/sphinx/source/whatsnew/v0.10.4.rst b/docs/sphinx/source/whatsnew/v0.10.4.rst
@@ -8,14 +8,15 @@ v0.10.4 (Anticipated March, 2024)
 Enhancements
 ~~~~~~~~~~~~
 * Added the Huld PV model used by PVGIS (:pull:`1940`)
+* Add :py:func:`pvlib.iotools.get_solargis` for retrieving Solargis
+  irradiance data. (:pull:`1969`)
 * Added function :py:func:`pvlib.shading.projected_solar_zenith_angle`,
   a common calculation in shading and tracking. (:issue:`1734`, :pull:`1904`)
 * Added :py:func:`~pvlib.iotools.get_solrad` for fetching irradiance data from
   the SOLRAD ground station network. (:pull:`1967`)
 * Added metadata parsing to :py:func:`~pvlib.iotools.read_solrad` to follow the standard iotools
   convention of returning a tuple of (data, meta). Previously the function only returned a dataframe. (:pull:`1968`)
 
-
 Bug fixes
 ~~~~~~~~~
 * Fixed an error in solar position calculations when using
@@ -33,6 +34,7 @@ Bug fixes
   ``temperature_model_parameters`` are specified on the passed ``system`` instead of on its ``arrays``. (:issue:`1759`).
 * :py:func:`pvlib.irradiance.ghi_from_poa_driesse_2023` now correctly makes use
   of the ``xtol`` argument. Previously, it was ignored. (:issue:`1970`, :pull:`1971`)
+* Fixed incorrect unit conversion of precipitable water used for the Solcast iotools functions.
 * :py:class:`~pvlib.modelchain.ModelChain.infer_temperature_model` now raises a more useful error when
   the temperature model cannot be inferred (:issue:`1946`)
 
@@ -49,6 +51,8 @@ Documentation
 
 Requirements
 ~~~~~~~~~~~~
+* Minimum version of pandas advanced from 0.25.0 to 1.3.0. (:pull:`1969`)
+* Minimum version of numpy advanced from 1.16.0 to 1.17.3. (:pull:`1969`)
 
 
 Contributors
@@ -59,5 +63,4 @@ Contributors
 * Cliff Hansen (:ghuser:`cwhanse`)
 * Roma Koulikov (:ghuser:`matsuobasho`)
 * Adam R. Jensen (:ghuser:`AdamRJensen`)
-* Kevin Anderson (:ghuser:`kandersolar`)
 * Peter Dudfield (:ghuser:`peterdudfield`)
diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py
@@ -34,3 +34,4 @@
 from pvlib.iotools.solcast import get_solcast_live  # noqa: F401
 from pvlib.iotools.solcast import get_solcast_historic  # noqa: F401
 from pvlib.iotools.solcast import get_solcast_tmy  # noqa: F401
+from pvlib.iotools.solargis import get_solargis  # noqa: F401
diff --git a/pvlib/iotools/solargis.py b/pvlib/iotools/solargis.py
@@ -0,0 +1,214 @@
+"""Functions to retrieve and parse irradiance data from Solargis."""
+
+import pandas as pd
+import requests
+from dataclasses import dataclass
+import io
+
+URL = 'https://solargis.info/ws/rest/datadelivery/request'
+
+
+TIME_RESOLUTION_MAP = {
+    5: 'MIN_5', 10: 'MIN_10', 15: 'MIN_15', 30:  'MIN_30', 60: 'HOURLY',
+    'PT05M': 'MIN_5', 'PT5M': 'MIN_5', 'PT10M': 'MIN_10', 'PT15M': 'MIN_15',
+    'PT30': 'MIN_30', 'PT60M': 'HOURLY', 'PT1H': 'HOURLY', 'P1D': 'DAILY',
+    'P1M': 'MONTHLY', 'P1Y': 'YEARLY'}
+
+
+@dataclass
+class ParameterMap:
+    solargis_name: str
+    pvlib_name: str
+    conversion: callable = lambda x: x
+
+
+# define the conventions between Solargis and pvlib nomenclature and units
+VARIABLE_MAP = [
+    # Irradiance (unit varies based on time resolution)
+    ParameterMap('GHI', 'ghi'),
+    ParameterMap('GHI_C', 'ghi_clear'),  # this is stated in documentation
+    ParameterMap('GHIc', 'ghi_clear'),  # this is used in practice
+    ParameterMap('DNI', 'dni'),
+    ParameterMap('DNI_C', 'dni_clear'),
+    ParameterMap('DNIc', 'dni_clear'),
+    ParameterMap('DIF', 'dhi'),
+    ParameterMap('GTI', 'poa_global'),
+    ParameterMap('GTI_C', 'poa_global_clear'),
+    ParameterMap('GTIc', 'poa_global_clear'),
+    # Solar position
+    ParameterMap('SE', 'solar_elevation'),
+    # SA -> solar_azimuth (degrees) (different convention)
+    ParameterMap("SA", "solar_azimuth", lambda x: x + 180),
+    # Weather / atmospheric parameters
+    ParameterMap('TEMP', 'temp_air'),
+    ParameterMap('TD', 'temp_dew'),
+    # surface_pressure (hPa) -> pressure (Pa)
+    ParameterMap('AP', 'pressure', lambda x: x*100),
+    ParameterMap('RH', 'relative_humidity'),
+    ParameterMap('WS', 'wind_speed'),
+    ParameterMap('WD', 'wind_direction'),
+    ParameterMap('INC', 'aoi'),  # angle of incidence of direct irradiance
+    # precipitable_water (kg/m2) -> precipitable_water (cm)
+    ParameterMap('PWAT', 'precipitable_water', lambda x: x/10),
+]
+
+METADATA_FIELDS = [
+    'issued', 'site name', 'latitude', 'longitude', 'elevation',
+    'summarization type', 'summarization period'
+]
+
+
+# Variables that use "-9" as nan values
+NA_9_COLUMNS = ['GHI', 'GHIc', 'DNI', 'DNIc', 'DIF', 'GTI', 'GIc', 'KT', 'PAR',
+                'PREC', 'PWAT', 'SDWE', 'SFWE']
+
+
+def get_solargis(latitude, longitude, start, end, variables, api_key,
+                 time_resolution, timestamp_type='center', tz='GMT+00',
+                 terrain_shading=True, url=URL, map_variables=True,
+                 timeout=30):
+    """
+    Retrieve irradiance time series data from Solargis.
+
+    The Solargis [1]_ API is described in [2]_.
+
+    Parameters
+    ----------
+    latitude: float
+        In decimal degrees, between -90 and 90, north is positive (ISO 19115)
+    longitude: float
+        In decimal degrees, between -180 and 180, east is positive (ISO 19115)
+    start : datetime-like
+        Start date of time series.
+    end : datetime-like
+        End date of time series.
+    variables : list
+        List of variables to request, see [2]_ for options.
+    api_key : str
+        API key.
+    time_resolution : str, {'PT05M', 'PT10M', 'PT15M', 'PT30', 'PT1H', 'P1D', 'P1M', 'P1Y'}
+        Time resolution as an integer number of minutes (e.g. 5, 60)
+        or an ISO 8601 duration string (e.g. "PT05M", "PT60M", "P1M").
+    timestamp_type : {'start', 'center', 'end'}, default: 'center'
+        Labeling of time stamps of the return data.
+    tz : str, default : 'GMT+00'
+        Timezone of `start` and `end` in the format "GMT+hh" or "GMT-hh".
+    terrain_shading : boolean, default: True
+        Whether to account for horizon shading.
+    url : str, default : :const:`pvlib.iotools.solargis.URL`
+        Base url of Solargis API.
+    map_variables : boolean, default: True
+        When true, renames columns of the Dataframe to pvlib variable names
+        where applicable. See variable :const:`VARIABLE_MAP`.
+    timeout : int or float, default: 30
+        Time in seconds to wait for server response before timeout
+
+    Returns
+    -------
+    data : DataFrame
+        DataFrame containing time series data.
+    meta : dict
+        Dictionary containing metadata.
+
+    Raises
+    ------
+    requests.HTTPError
+        A message from the Solargis server if the request is rejected
+
+    Notes
+    -----
+    Each XML request is limited to retrieving 31 days of data.
+
+    The variable units depends on the time frequency, e.g., the unit for
+    sub-hourly irradiance data is :math:`W/m^2`, for hourly data it is
+    :math:`Wh/m^2`, and for daily data it is :math:`kWh/m^2`.
+
+    References
+    ----------
+    .. [1] `Solargis <https://solargis.com>`_
+    .. [2] `Solargis API User Guide
+       <https://solargis.atlassian.net/wiki/spaces/public/pages/7602367/Solargis+API+User+Guide>`_
+
+    Examples
+    --------
+    >>> # Retrieve two days of irradiance data from Solargis
+    >>> data, meta = response = pvlib.iotools.get_solargis(
+    >>>     latitude=48.61259, longitude=20.827079,
+    >>>     start='2022-01-01', end='2022-01-02',
+    >>>     variables=['GHI', 'DNI'], time_resolution='PT05M', api_key='demo')
+    """  # noqa: E501
+    # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
+    start = pd.to_datetime(start)
+    end = pd.to_datetime(end)
+
+    headers = {'Content-Type': 'application/xml'}
+
+    # Solargis recommends creating a unique site_id for each location request.
+    # The site_id does not impact the data retrieval and is used for debugging.
+    site_id = f"latitude_{latitude}_longitude_{longitude}"
+
+    request_xml = f'''<ws:dataDeliveryRequest
+    dateFrom="{start.strftime('%Y-%m-%d')}"
+    dateTo="{end.strftime('%Y-%m-%d')}"
+    xmlns="http://geomodel.eu/schema/data/request"
+    xmlns:ws="http://geomodel.eu/schema/ws/data"
+    xmlns:geo="http://geomodel.eu/schema/common/geo"
+    xmlns:pv="http://geomodel.eu/schema/common/pv"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <site id="{site_id}" name="" lat="{latitude}" lng="{longitude}">
+    </site>
+    <processing key="{' '.join(variables)}"
+    summarization="{TIME_RESOLUTION_MAP.get(time_resolution, time_resolution).upper()}"
+    terrainShading="{str(terrain_shading).lower()}">
+    <timestampType>{timestamp_type.upper()}</timestampType>
+    <timeZone>{tz}</timeZone>
+    </processing>
+    </ws:dataDeliveryRequest>'''  # noqa: E501
+
+    response = requests.post(url + "?key=" + api_key, headers=headers,
+                             data=request_xml.encode('utf8'), timeout=timeout)
+
+    if response.ok is False:
+        raise requests.HTTPError(response.json())
+
+    # Parse metadata
+    header = pd.read_xml(io.StringIO(response.text),  parser='etree')
+    meta_lines = header['metadata'].iloc[0].split('#')
+    meta_lines = [line.strip() for line in meta_lines]
+    meta = {}
+    for line in meta_lines:
+        if ':' in line:
+            key = line.split(':')[0].lower()
+            if key in METADATA_FIELDS:
+                meta[key] = ':'.join(line.split(':')[1:])
+    meta['latitude'] = float(meta['latitude'])
+    meta['longitude'] = float(meta['longitude'])
+    meta['altitude'] = float(meta.pop('elevation').replace('m a.s.l.', ''))
+
+    # Parse data
+    data = pd.read_xml(io.StringIO(response.text), xpath='.//doc:row',
+                       namespaces={'doc': 'http://geomodel.eu/schema/ws/data'},
+                       parser='etree')
+    data.index = pd.to_datetime(data['dateTime'])
+    # when requesting one variable, it is necessary to convert dataframe to str
+    data = data['values'].astype(str).str.split(' ', expand=True)
+    data = data.astype(float)
+    data.columns = header['columns'].iloc[0].split()
+
+    # Replace "-9" with nan values for specific columns
+    for variable in data.columns:
+        if variable in NA_9_COLUMNS:
+            data[variable] = data[variable].replace(-9, pd.NA)
+
+    # rename and convert variables
+    if map_variables:
+        for variable in VARIABLE_MAP:
+            if variable.solargis_name in data.columns:
+                data.rename(
+                    columns={variable.solargis_name: variable.pvlib_name},
+                    inplace=True
+                )
+                data[variable.pvlib_name] = data[
+                    variable.pvlib_name].apply(variable.conversion)
+
+    return data, meta
diff --git a/pvlib/iotools/solcast.py b/pvlib/iotools/solcast.py
@@ -35,7 +35,7 @@ class ParameterMap:
         "azimuth", "solar_azimuth", lambda x: -x % 360
     ),
     # precipitable_water (kg/m2) -> precipitable_water (cm)
-    ParameterMap("precipitable_water", "precipitable_water", lambda x: x*10),
+    ParameterMap("precipitable_water", "precipitable_water", lambda x: x/10),
     # zenith -> solar_zenith
     ParameterMap("zenith", "solar_zenith"),
     # clearsky
diff --git a/pvlib/tests/iotools/test_solargis.py b/pvlib/tests/iotools/test_solargis.py
diff --git a/pvlib/tests/iotools/test_solcast.py b/pvlib/tests/iotools/test_solcast.py
diff --git a/pyproject.toml b/pyproject.toml
