From c423758c8bdc85070ab1e7ff0a50f0089169138e Mon Sep 17 00:00:00 2001 From: veenstrajelmer <60435591+veenstrajelmer@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:55:21 +0100 Subject: [PATCH] 56 create ddlpy 030 release (#81) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improved cli docs * update docs * Bump version: 0.2.1 → 0.3.0 --- .bumpversion.cfg | 2 +- .github/workflows/pytest.yml | 2 +- CONTRIBUTING.rst | 17 ++++++++ HISTORY.rst | 2 +- README.md | 44 ++++++++++----------- ddlpy/__init__.py | 2 +- ddlpy/cli.py | 75 +++++++++++++++++++----------------- docs/index.rst | 4 -- docs/installation.rst | 51 ------------------------ docs/modules.rst | 1 + examples/minimal_example.py | 23 +++++------ pyproject.toml | 9 ++--- tests/test_cli.py | 4 +- 13 files changed, 98 insertions(+), 138 deletions(-) delete mode 100644 docs/installation.rst diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a402bef..6d3fc79 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.1 +current_version = 0.3.0 commit = True tag = True diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index da8fdf3..685c62a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[dev,waterinfo] + python -m pip install -e .[dev,netcdf] - name: list env contents run: | pip list diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c1026bf..05b477b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -126,3 +126,20 @@ $ git push $ git push --tags Travis will then deploy to PyPI if tests pass. + + +Releasing (non-travis) +---------------------- + +- make sure the ``main`` branch is up to date +- bump the versionnumber with ``bumpversion minor`` +- update heading (including date) in ``HISTORY.rst`` +- run testbank +- local check with: ``python -m build`` and ``twine check dist/*`` +- push+merge all changes +- create a new release at https://github.com/Deltares/dfm_tools/releases/new +- click ``choose a tag`` and type v+versionnumber (e.g. ``v0.3.0``), click ``create new tag on publish`` +- set the release title to the tagname (e.g. ``v0.3.0``) +- click `Generate release notes` +- if all is set, click ``Publish release`` +- a release is created and published on PyPI by the github action diff --git a/HISTORY.rst b/HISTORY.rst index cd2e39f..4ffe5cb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,7 +2,7 @@ History ======= -UNRELEASED +0.3.0 (2023-03-13) ------------------ * improved nan filtering of measurements in https://github.com/deltares/ddlpy/pull/30 * add `ddlpy.measurements_available()` check in https://github.com/deltares/ddlpy/pull/33 and https://github.com/deltares/ddlpy/pull/58 diff --git a/README.md b/README.md index 79fa4f0..4d4376b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# ddlpy - [![pypi-image](https://img.shields.io/pypi/v/ddlpy.svg)](https://pypi.python.org/pypi/ddlpy) [![pytest](https://github.com/Deltares/ddlpy/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/Deltares/ddlpy/actions/workflows/pytest.yml) [![codecov](https://img.shields.io/codecov/c/github/deltares/ddlpy.svg?style=flat-square)](https://app.codecov.io/gh/deltares/ddlpy?displayType=list) @@ -7,49 +5,49 @@ [![Supported versions](https://img.shields.io/pypi/pyversions/ddlpy.svg)](https://pypi.org/project/ddlpy) [![Downloads](https://img.shields.io/pypi/dm/ddlpy.svg)](https://pypistats.org/packages/ddlpy) -(D)ata (D)istributie (L)aag is a service from Rijkswaterstaat for distributing water quantity data. This package provides an API for python. +# ddlpy -Documentation: +(D)ata (D)istributie (L)aag is a service from Rijkswaterstaat for distributing water quantity data. This package provides an API for python. See also https://github.com/wstolte/rwsapi for the R API. # Install -This text will be updated soon and a new pypi release will also happen soon. -The latest ddlpy PyPI release is outdated, but it can be installed with: +If you had ddlpy installed before please uninstall it first, since the package was renamed from rws-ddlpy to ddlpy on PyPI: + + pip uninstall rws-ddlpy -y - pip install rws-ddlpy +Install the latest ddlpy PyPI release with (extra dependencies between `[]` are optional): -The newest version is currently installed directly from github with: + pip install ddlpy[netcdf,examples] - pip install git+https://github.com/deltares/ddlpy +# Examples + +Documentation: -In the folder examples you will find the following files: +In the examples/notebooks folders you will find the following examples to get you started: -* minimal example.py -> minimal code to retrieve data. +* [minimal_example.py](https://github.com/Deltares/ddlpy/blob/main/examples/minimal_example.py) -> minimal code to retrieve data. -* 1_get_data_from_water_info_parallel.py -> Code to retrieve a bulk of observations per parameter and per station. +* [retrieve_parallel_to_netcdf.py](https://github.com/Deltares/ddlpy/blob/main/examples/retrieve_parallel_to_netcdf.py) -> Code to retrieve a bulk of observations and write to netcdf files for each station. -The output of this code is the data in csv format. +* [measurements.ipynb](https://github.com/Deltares/ddlpy/blob/main/notebooks/measurements.ipynb) -> interactive notebook to subset/inspect locations and download/plot measurements -* 2_get_netcdf.py -> Code to transform the csv files run in the previous script into netcdf files. +* [waterinfo.ipynb](https://github.com/Deltares/ddlpy/blob/main/notebooks/waterinfo.ipynb) -> interactive notebook to read csv's obained from waterinfo.rws.nl -More detailed explanation on the usage of these codes are inside the `notebooks` directory. # Run ddlpy from console -You can also run ddlpy from the console. The options you can use are the following: -* Write locations metadata to output file, given input station codes and parameter codes: +With `ddlpy locations` you can generate a (subsetted) locations.json file, for instance: - ddlpy locations + ddlpy locations --quantity WATHTE --station HOEKVHLD -To get access to the help menu, type in a terminal: ddlpy locations --help. +With `ddlpy measurements` you can obtain measurements for locations/parameters in an existing locations.json, for instance: -* Obtain measurements from json file containing locations and codes: + ddlpy measurements 2023-01-01 2023-01-03 - ddlpy measurements -To get access to the help menu, type in a terminal: ddlpy measurements --help. +# Something broke? -IMPORTANT: You can not run `ddlpy measurements` before running `ddlpy locations`, unless you already have a .json file listing the stations and the parameters you need data from. +Check the [status of the DDL](https://rijkswaterstaatdata.nl/waterdata/#hfd2f5e23-5092-4169-9f36-41e9734e7d87) (at the *Updates* heading). If you have a suggestion or found a bug in ddlpy, please [create an issue](https://github.com/Deltares/ddlpy/issues). diff --git a/ddlpy/__init__.py b/ddlpy/__init__.py index 665e30c..aa9cc87 100644 --- a/ddlpy/__init__.py +++ b/ddlpy/__init__.py @@ -4,7 +4,7 @@ __author__ = """Fedor Baart""" __email__ = 'fedor.baart@deltares.nl' -__version__ = '0.2.1' +__version__ = '0.3.0' from ddlpy.ddlpy import locations from ddlpy.ddlpy import (measurements, diff --git a/ddlpy/cli.py b/ddlpy/cli.py index 8d96721..83f3091 100644 --- a/ddlpy/cli.py +++ b/ddlpy/cli.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- -"""Console script for ddlpy.""" +""" +Console script for ddlpy. + - ``ddlpy --help`` + - ``ddlpy locations --help`` + - ``ddlpy measurements --help`` +""" import sys import logging import click @@ -24,63 +29,63 @@ def cli(verbose, args=None): @cli.command() @click.option( '--output', - help='output of locations json file', + help='the locations json filename that will be created', default='locations.json' ) @click.option( - '--quantity', - help='Grootheid code', + '--station', + help='Station codes, e.g. HOEKVHLD', multiple=True ) @click.option( - '--quality', - help='Hoedanigheid code', + '--grootheid-code', + help='Grootheid code, e.g. WATHTE', multiple=True ) @click.option( - '--unit', - help='Eenheid code', + '--groepering-code', + help='Groepering code, e.g. NVT', multiple=True ) @click.option( - '--parameter-code', - help='Parameter code', + '--hoedanigheid-code', + help='Hoedanigheid code, e.g. NAP', multiple=True ) @click.option( - '--compartment-code', - help='Compartment code', + '--eenheid-code', + help='Eenheid code, e.g. cm', multiple=True ) @click.option( - '--station', - help='Station codes', + '--parameter-code', + help='Parameter code', multiple=True ) @click.option( - '--format', - default='json', - help='output file format. Must be json', - type=click.Choice(['json'], case_sensitive=True) + '--compartment-code', + help='Compartment code, e.g. OW', + multiple=True ) def locations(output, station, - quantity, - quality, - unit, + grootheid_code, + groepering_code, + hoedanigheid_code, + eenheid_code, parameter_code, - compartment_code, - format): + compartment_code): """ - Write locations metadata to output file, given input codes. + Subset locations dataframe based on input codes and write locations.json. """ locations_df = ddlpy.locations() stations = station - quantities = {'Grootheid.Code': list(quantity), - 'Hoedanigheid.Code': list(quality), - 'Eenheid.Code': list(unit), + quantities = {'Grootheid.Code': list(grootheid_code), + 'Groepering.Code': list(groepering_code), + 'Hoedanigheid.Code': list(hoedanigheid_code), + 'Eenheid.Code': list(eenheid_code), 'Parameter.Code': list(parameter_code), 'Compartiment.Code': list(compartment_code) } @@ -96,11 +101,8 @@ def locations(output, selected.reset_index(inplace= True) - if format == 'json': - output= output.split('.')[0] # make sure that extension is always json - selected.to_json(output+'.json', orient='records') - else: - raise ValueError('Unexpected format {}'.format(format)) + output= output.split('.')[0] # make sure that extension is always json + selected.to_json(output+'.json', orient='records') # Another command to get the measurements from locations @cli.command() @@ -117,7 +119,9 @@ def locations(output, ) def measurements(locations, start_date, end_date): """ - Obtain measurements from file with locations and codes + Obtain measurements from file with locations and codes. + The arguments start_date and end_date should be formatted + like "YYYY-MM-DD" or something else that `pandas.Timestamp` understands. """ try: locations_df = pd.read_json(locations, orient='records') @@ -136,11 +140,12 @@ def measurements(locations, start_date, end_date): cc = selected['Compartiment.Code'] ec = selected['Eenheid.Code'] gc = selected['Grootheid.Code'] + grc = selected['Groepering.Code'] hc = selected['Hoedanigheid.Code'] pc = selected['Parameter.Code'] - measurements.to_csv('%s_%s_%s_%s_%s_%s.csv' % - (station, cc, ec, gc, hc, pc)) + measurements.to_csv('%s_%s_%s_%s_%s_%s_%s.csv' % + (station, cc, ec, gc, grc, hc, pc)) else: print('No Data of station %s were retrieved from Water Info' % selected['Code']) diff --git a/docs/index.rst b/docs/index.rst index 7b7b537..bba681c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,13 +1,9 @@ -ddlpy documentation -=================== - .. mdinclude:: ../README.md .. toctree:: :titlesonly: :hidden: - installation usage modules contributing diff --git a/docs/installation.rst b/docs/installation.rst deleted file mode 100644 index fede821..0000000 --- a/docs/installation.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. highlight:: shell - -============ -Installation -============ - - -Stable release --------------- - -To install Data Distributie Laag. Service from Rijkswaterstaat for distributing water quantity data., run this command in your terminal: - -.. code-block:: console - - $ pip install ddlpy - -This is the preferred method to install Data Distributie Laag. Service from Rijkswaterstaat for distributing water quantity data., as it will always install the most recent stable release. - -If you don't have `pip`_ installed, this `Python installation guide`_ can guide -you through the process. - -.. _pip: https://pip.pypa.io -.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ - - -From sources ------------- - -The sources for Data Distributie Laag. Service from Rijkswaterstaat for distributing water quantity data. can be downloaded from the `Github repo`_. - -You can either clone the public repository: - -.. code-block:: console - - $ git clone git://github.com/deltares/ddlpy - -Or download the `tarball`_: - -.. code-block:: console - - $ curl -OL https://github.com/deltares/ddlpy/tarball/master - -Once you have a copy of the source, you can install it with: - -.. code-block:: console - - $ python setup.py install - - -.. _Github repo: https://github.com/deltares/ddlpy -.. _tarball: https://github.com/deltares/ddlpy/tarball/master diff --git a/docs/modules.rst b/docs/modules.rst index cb861b4..f98de20 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -9,6 +9,7 @@ ddlpy.cli module :undoc-members: :show-inheritance: + ddlpy module --------------- diff --git a/examples/minimal_example.py b/examples/minimal_example.py index 715332a..b3469f0 100644 --- a/examples/minimal_example.py +++ b/examples/minimal_example.py @@ -1,13 +1,11 @@ """ -This is a minimal example on how to retrieve data from water info. -Make sure to set up a path to store the resulting dataframe. Also, decomment -the line of code +This is a minimal example on how to retrieve data from the DDL with ddlpy. """ -from ddlpy import ddlpy +import ddlpy import datetime as dt -# get all locations +# get the dataframe with locations and their available parameters locations = ddlpy.locations() #select a set of parameters @@ -22,17 +20,14 @@ selected = locations.loc[bool_stations & bool_grootheid & bool_groepering & bool_hoedanigheid] -# Obtain measurements per parameter row -index = 1 -location = selected.reset_index().iloc[index] +start_date = dt.datetime(2023, 1, 1) +end_date = dt.datetime(2023, 1, 15) -start_date = dt.datetime(2015, 1, 1) # also inputs for the code -end_date = dt.datetime(2015, 6, 1) -measurements = ddlpy.measurements(location, start_date=start_date, end_date=end_date) +# provide a single row of the locations dataframe to ddlpy.measurements +measurements = ddlpy.measurements(selected.iloc[0], start_date=start_date, end_date=end_date) -if (len(measurements) > 0): +if not measurements.empty: print('Data was found in Waterbase') - #measurements.to_csv("%s_%s_%s.csv"%(location.Code, code, unit), index= False) - measurements.plot(y="Meetwaarde.Waarde_Numeriek") + measurements.plot(y="Meetwaarde.Waarde_Numeriek", linewidth=0.8) else: print('No Data!') diff --git a/pyproject.toml b/pyproject.toml index de84729..93ad172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ddlpy" -version = "0.2.1" +version = "0.3.0" maintainers = [ { name = "Fedor Baart", email = "fedor.baart@deltares.nl"}, { name = "Jelmer Veenstra", email = "jelmer.veenstra@deltares.nl"}, @@ -44,14 +44,13 @@ Code = "https://github.com/deltares/ddlpy" Issues = "https://github.com/deltares/ddlpy/issues" [project.optional-dependencies] -dev = [ #TODO: maybe remove some of these +dev = [ "bump2version>=0.5.11", "flake8", "pytest>=3.8.2", "pytest-cov", "twine", "build", - "watchdog>=0.9.0", "flake8>=3.5.0", "tox>=3.5.2", "twine>=1.12.1", @@ -66,8 +65,8 @@ examples = [ "jupyter", "notebook", ] -waterinfo = [ - "xarray", +netcdf = [ + "xarray[io]", ] [project.scripts] diff --git a/tests/test_cli.py b/tests/test_cli.py index 9d3c830..50cd628 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,7 +19,7 @@ def test_command_line_interface(): assert help_result.exit_code == 0 assert 'Show this message and exit.' in help_result.output - locations_command = 'locations --quantity WATHTE --station HOEKVHLD' + locations_command = 'locations --grootheid-code WATHTE --station HOEKVHLD' locations_result = runner.invoke(cli.cli, locations_command.split()) assert locations_result.exit_code == 0 assert os.path.exists("locations.json") @@ -27,4 +27,4 @@ def test_command_line_interface(): measurements_command = 'measurements 2023-01-01 2023-01-03' measurements_result = runner.invoke(cli.cli, measurements_command.split()) assert measurements_result.exit_code == 0 - assert os.path.exists("HOEKVHLD_OW_cm_WATHTE_NAP_NVT.csv") + assert os.path.exists("HOEKVHLD_OW_cm_WATHTE_NVT_NAP_NVT.csv")