diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 95858e0..7d929f1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.4.2 +current_version = 2.0.0b1 [comment] comment = The contents of this file cannot be merged with that of pyproject.toml until https://github.com/c4urself/bump2version/issues/42 is resolved diff --git a/.github/.codecov.yml b/.github/.codecov.yml index cd7a776..9057cd1 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -1,19 +1,15 @@ -# -# This codecov.yml is the default configuration for -# all repositories on Codecov. You may adjust the settings -# below in your own codecov.yml in your repository. -# +comment: + behavior: default + layout: header, diff coverage: precision: 2 + range: + - 70.0 + - 100.0 round: down - range: 70...100 status: - # Learn more at https://codecov.io/docs#yaml_default_commit_status - project: true - patch: true changes: false + patch: true + project: true ignore: - - "tests" -comment: - layout: header, diff - behavior: default # update if exists else create new +- ^tests.* diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml index af0c4a3..5f82dec 100644 --- a/.github/workflows/test_codecov.yml +++ b/.github/workflows/test_codecov.yml @@ -22,10 +22,11 @@ jobs: - name: Generate test coverage run: hatch run coverage - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: directory: ./coverage/reports/ files: ./cov.xml env_vars: python-version fail_ci_if_error: true verbose: true + token: ${{ secrets.CODECOV_TOKEN }} # required diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index a46fe40..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -repos: -- repo: local - hooks: - - id: run-formatter - name: run-formatter - entry: hatch run format - language: system - types: [python] diff --git a/CITATION.cff b/CITATION.cff index a54f85f..074058c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -97,4 +97,4 @@ license: Apache-2.0 message: "If you use this software, please cite it using these metadata." repository-code: "https://github.com/ewatercycle/era5cli" title: era5cli -version: "1.4.2" +version: "2.0.0b1" diff --git a/README.md b/README.md index b4897b2..6473ab8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,21 @@ Logo +> [!IMPORTANT] +> The old Climate Data Store (CDS) will be shut down on 3 September 2024. +> All era5cli versions up to v1.4.2 will no longer work. +> +> For more information see: https://forum.ecmwf.int/t/the-new-climate-data-store-beta-cds-beta-is-now-live/3315 +> +> +> To continue using era5cli, you will need to re-register at ECMWF and get a new API key, +> and transition to the era5cli v2 beta. This can be installed with: +> `pip install era5cli==2.0.0b1` + +> [!WARNING] +> netCDF files from the new Climate Data Store Beta are not formatted the same as the +> old CDS. Some variables might be missing. +> See the open issue [here](https://github.com/eWaterCycle/era5cli/issues/165), as well as the [ECMWF discussion forum](https://forum.ecmwf.int/). + [![github license badge](https://img.shields.io/github/license/eWaterCycle/era5cli)](https://github.com/eWaterCycle/era5cli) [![rsd badge](https://img.shields.io/badge/RSD-era5cli-blue)](https://research-software-directory.org/software/era5cli) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 955b6de..754962c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## 2.0.0b1 - 2024-08-30 + +**Changed:** + + - The `splitmonths` argument now defaults to `True` for hourly requests. To not split requests by year, add `--splitmonths False`. + +**Fixed:** + + - Added support for the beta-CDS + - For authentication, the new `cads-api-client` is used, instead of a dummy request. This should avoid the dummy requests appearing in the user's queue. + +**Removed:** + + - the deprecated `orography` variable. Use `geopotential` instead. + - the deprecated `--prelimbe` argument. This one has not been required anymore, as the back-extension is part of the normal dataset now. + +**Dev changes:** + + - The pre-commit hook has been removed. Pre-commit does not play well with hatch: it would need to be installed system-wide. No hatch-specific hooks are available. + + ## 1.4.2 - 2023-12-12 **Fixed:** diff --git a/docs/formulating_requests.md b/docs/formulating_requests.md index ef49256..d794ab0 100644 --- a/docs/formulating_requests.md +++ b/docs/formulating_requests.md @@ -20,13 +20,12 @@ era5cli hourly \ --variables 2m_temperature 2m_dewpoint_temperature \ --startyear 2000 \ --endyear 2020 \ - --splitmonths True \ --area 53.6 3.3 50.7 7.5 ``` This request asks for *hourly* data of the ERA5-*Land* dataset, more specifically the *2m_temperature* and *2m_dewpoint_temperature* variables. -Additionally, data from the year *2000* up to (and including) *2020* is requested, with the final files being *split up by months*. +Additionally, data from the year *2000* up to (and including) *2020* is requested. Lastly, an *area* is extracted from the dataset (in this case only the Netherlands). ### Using the info command diff --git a/docs/general_development.md b/docs/general_development.md index 03d0041..73ea613 100644 --- a/docs/general_development.md +++ b/docs/general_development.md @@ -80,17 +80,6 @@ hatch run format This will apply the `black` and `isort` formatting, and then check the code style. -??? tip "Using pre-commit" - For pre-commit users, a pre-commit configuration has been added. This hook will execute the `hatch run format` command. - - After installing pre-commit in your python environment (`pip install pre-commit`), you can do - ``` - pre-commit install - ``` - to set up the git hook scripts. - - For more information, see the [pre-commit website](https://pre-commit.com/). - ## Generating the documentation To view the documentation locally, simply run the following command: diff --git a/docs/getting_started.md b/docs/getting_started.md index 8a1edc8..31e0f42 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -43,10 +43,10 @@ After activating your account, **login** on the CDS website, go to your **profil To configure era5cli to use these keys, open up the environment you installed era5cli in, and do: ```sh -era5cli config --uid ID_NUMBER --key "KEY" +era5cli config --key "KEY" ``` -*Where ID_NUMBER is your user ID (e.g. 123456) and "KEY" is your API key, inside double quotes (e.g. "4s215sgs-2dfa-6h34-62h2-1615ad163414").* +*Where "KEY" is your API key, inside double quotes (e.g. "4s215sgs-2dfa-6h34-62h2-1615ad163414").* After running this command your ID and key are validated and stored inside your home folder, under `.config/era5cli/cds_key.txt`. diff --git a/docs/index.md b/docs/index.md index 8163c8b..8f22597 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,22 @@ A command line interface to download ERA5 data from the [Copernicus Climate Data Store](https://climate.copernicus.eu/). +???+ note + The old Climate Data Store (CDS) will be shut down on 3 September 2024. + All era5cli versions up to v1.4.2 will no longer work. + + For more information see: https://forum.ecmwf.int/t/the-new-climate-data-store-beta-cds-beta-is-now-live/3315 + + To continue using era5cli, you will need to re-register at ECMWF and get a new API key, + and transition to the era5cli v2 beta. This can be installed with: + `pip install era5cli==2.0.0b1` + +???+ warning + netCDF files from the new Climate Data Store Beta are not formatted the same as the + old CDS. Some variables might be missing. + + See the open issue [here](https://github.com/eWaterCycle/era5cli/issues/165), as well as the [ECMWF discussion forum](https://forum.ecmwf.int/). + With era5cli you can: - Download meteorological data in GRIB/NetCDF, including ERA5 data from the preliminary back extension, and ERA5-Land data. diff --git a/era5cli/__version__.py b/era5cli/__version__.py index d434852..28f7ed3 100644 --- a/era5cli/__version__.py +++ b/era5cli/__version__.py @@ -26,4 +26,4 @@ "Bart Schilperoort", ) __email__ = "ewatercycle@esciencecenter.nl" -__version__ = "1.4.2" +__version__ = "2.0.0b1" diff --git a/era5cli/args/common.py b/era5cli/args/common.py index 86093ad..3282b3b 100644 --- a/era5cli/args/common.py +++ b/era5cli/args/common.py @@ -24,7 +24,6 @@ def add_common_args(argument_parser: ArgumentParser) -> None: --threads, --ensemble, --dryrun, - --prelimbe, --land, --area, --overwrite @@ -188,23 +187,6 @@ def add_common_args(argument_parser: ArgumentParser) -> None: ), ) - argument_parser.add_argument( - "--prelimbe", - action="store_true", - default=False, - help=textwrap.dedent( - """ - Whether to download the preliminary back extension - (1950-1978). Note that when `--prelimbe` is used, - `--startyear` and `--endyear` should be set - between 1950 and 1978. Please, be aware that - ERA5 data is available from 1940. - `--prelimbe` is incompatible with `--land` - - """ - ), - ) - argument_parser.add_argument( "--land", action="store_true", @@ -215,7 +197,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: dataset. Note that the ERA5-Land dataset starts in 1950. `--land` is incompatible with the use of - `--prelimbe` and `--ensemble` + `--ensemble` """ ), @@ -287,9 +269,7 @@ def construct_year_list(args): # check whether correct years have been entered for year in (args.startyear, endyear): - if args.prelimbe: - assert 1950 <= year <= 1978, "year should be between 1950 and 1978" - elif args.land: + if args.land: assert ( 1950 <= year <= datetime.now().year ), "for ERA5-Land, year should be between 1950 and present" diff --git a/era5cli/args/config.py b/era5cli/args/config.py index 2e947b2..b986bab 100644 --- a/era5cli/args/config.py +++ b/era5cli/args/config.py @@ -10,7 +10,6 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: Adds the 'config' parser with the following arguments: --show - --uid --key --url @@ -27,7 +26,7 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: This will create a config file in your home directory, in folder named ".config". The CDS URL, your UID and the CDS keys will be stored here. - To find your key and UID, go to https://cds.climate.copernicus.eu/ and + To find your key, go to https://beta-cds.climate.copernicus.eu/ and login with your email and password. Then go to your user profile (top right). @@ -54,16 +53,6 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: ), ) - config.add_argument( - "--uid", - type=str, - help=textwrap.dedent( - """ - Your CDS User ID, e.g.: 123456 - """ - ), - ) - config.add_argument( "--key", type=str, @@ -87,6 +76,18 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: ), ) + config.add_argument( + "--uid", + type=str, + required=False, + default="", + help=textwrap.dedent( + """ + DO NOT USE: deprecated due to changes in the CDS API" + """ + ), + ) + class InputError(Exception): "Raised when a user inputs an invalid combination of arguments." @@ -101,27 +102,22 @@ def run_config(args): Args: args: Arguments collected by argparse - - Returns: - True """ - if args.show and any((args.uid, args.key)): + if len(args.uid) > 0: + msg = ( + "The `uid` argument is deprecated.\n" + "The new CDS API does not use UIDs anymore." + ) + raise InputError(msg) + + if args.show and args.key is not None: raise InputError("Either call `show` or set the key. Not both.") - if not args.show and (args.uid is None or args.key is None): - raise InputError("Both the UID and the key are required inputs.") + if not args.show and args.key is None: + raise InputError("Your CDS API key is a required input.") if args.show: - url, fullkey = key_management.load_era5cli_config() - uid, key = fullkey.split(":") + url, key = key_management.load_era5cli_config() print( - "Contents of .config/era5cli.txt:\n" - f" uid: {uid}\n" - f" key: {key}\n" - f" url: {url}\n" + "Contents of .config/era5cli.txt:\n" f" key: {key}\n" f" url: {url}\n" ) - return True - - return key_management.set_config( - url=args.url, - uid=args.uid, - key=args.key, - ) + else: + key_management.set_config(args.url, args.key) diff --git a/era5cli/args/periods.py b/era5cli/args/periods.py index a049590..092af1b 100644 --- a/era5cli/args/periods.py +++ b/era5cli/args/periods.py @@ -1,5 +1,4 @@ import argparse -import logging import textwrap from era5cli import utils @@ -73,15 +72,13 @@ def add_period_args(subparsers, common): splitmonths.add_argument( "--splitmonths", type=lambda x: bool(utils.strtobool(x)), # type=bool doesn't work. - default=None, # To be set to True in the future + default=True, help=textwrap.dedent( """ - When downloading hourly data, use: - `--splitmonths True` to split requests and files - by month, and add the month to the filename. - - Defaults to `False`, but will default to `True` in - a future release. + By default when downloading hourly data requests are split + by months. + To suppress this behavior, use: `--splitmonths False` to have yearly + files. """ ), ) @@ -175,17 +172,7 @@ def set_period_args(args): hours = args.synoptic elif args.command == "hourly": synoptic = None - if args.splitmonths is None: - splitmonths = False - logging.warning( - "\n The argument --splitmonths was not used. However, in a future " - "\n version this flag will default to `True`. To avoid this, either" - "\n use `--splitmonths True` and update your workflow accordingly," - "\n or set --splitmonths to False." - ) - else: - splitmonths: bool = args.splitmonths - + splitmonths: bool = args.splitmonths statistics: bool = args.statistics if statistics: assert args.ensemble, ( diff --git a/era5cli/cli.py b/era5cli/cli.py index 5123135..d4a5c6f 100644 --- a/era5cli/cli.py +++ b/era5cli/cli.py @@ -67,7 +67,6 @@ def _execute(input_args: argparse.Namespace) -> True: threads=input_args.threads, splitmonths=splitmonths, merge=input_args.merge, - prelimbe=input_args.prelimbe, land=input_args.land, overwrite=input_args.overwrite, dashed_vars=input_args.dashed_varname, diff --git a/era5cli/fetch.py b/era5cli/fetch.py index 1da3f7d..d6a4d9b 100644 --- a/era5cli/fetch.py +++ b/era5cli/fetch.py @@ -78,15 +78,11 @@ class Fetch: or make the request to start downloading the data. `dryrun = True` will print the request to stdout. By default, the data will be downloaded. - prelimbe: bool - Whether to download the preliminary back extension (1950-1978). - Note that in this case, `years` must be between 1950 and - 1978. `prelimbe = True` is incompatible with `land = True`. land: bool Whether to download data from the ERA5-Land dataset. Note that the ERA5-Land dataset starts in 1981. `land = True` is incompatible with the use of - `prelimbe = True` and `ensemble = True`. + `ensemble = True`. overwrite: bool Whether to overwrite existing files or not. Setting `overwrite = True` will make @@ -117,7 +113,6 @@ def __init__( splitmonths=False, merge=False, threads=None, - prelimbe=False, land=False, overwrite=False, dashed_vars=False, @@ -173,9 +168,6 @@ def __init__( """bool: Whether to get monthly averaged by hour of day (synoptic=True) or monthly means of daily means (synoptic=False).""" - self.prelimbe = prelimbe - """bool: Whether to select from the ERA5 preliminary back - extension which supports years from 1950 to 1978""" self.land = land """bool: Whether to download from the ERA5-Land dataset.""" @@ -186,24 +178,12 @@ def __init__( files, or the normal names.""" if self.merge and self.splitmonths: - raise ValueError( - "\nThe commands '--merge' and '--splitmonths' are not compatible with" - "\neach other. Please pick one of the two." - ) - - if self.prelimbe: - logging.warning( - "\n The years of the ERA5 preliminary back extension (1950 - 1978) are" - "\n now included in the main ERA5 products. The `--prelimbe` argument" - "\n will be deprecated in a future release." - "\n Please update your workflow accordingly." - ) + self.splitmonths = False vars = list(self.variables) # Use list() to avoid copying by reference if "geopotential" in vars and pressurelevels == ["surface"]: vars.remove("geopotential") if any([var in ref.PLVARS for var in vars]): - print(pressurelevels) self._check_levels() if self.period == "hourly" and request_too_large(self): @@ -371,18 +351,6 @@ def _product_type(self): if self.synoptic: producttype += "_by_hour_of_day" - if not self.prelimbe: - return producttype - - # Prelimbe has deviating product types for monthly data - if self.ensemble: - producttype = "members-" - else: - producttype = "reanalysis-" - if self.synoptic: - producttype += "synoptic-monthly-means" - else: - producttype += "monthly-means-of-daily-means" return producttype def _check_levels(self): @@ -470,14 +438,6 @@ def _build_name(self, variable): if self.land: name += "-land" - elif variable == "orography": - variable = "geopotential" - name += "-single-levels" - logging.warning( - "\n The variable 'orography' has been deprecated by CDS. Use" - "\n `--variables geopotential --levels surface` going forward." - "\n The current query has been changed accordingly." - ) elif self.pressure_levels == ["surface"]: name += "-single-levels" elif variable in ref.PLVARS: @@ -490,13 +450,6 @@ def _build_name(self, variable): if self.period == "monthly": name += "-monthly-means" - if self.prelimbe: - if self.land: - raise ValueError( - "Back extension not available for ERA5-Land. " - "ERA5-Land data is available from 1950 on." - ) - name += "-preliminary-back-extension" return name, variable def _build_request(self, variable, years, months=None): diff --git a/era5cli/inputref.py b/era5cli/inputref.py index 16c12ef..5dc362f 100644 --- a/era5cli/inputref.py +++ b/era5cli/inputref.py @@ -13,19 +13,15 @@ Single levels (hourly) variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * reanalysis-era5-single-levels -* reanalysis-era5-single-levels-preliminary-back-extension Single levels (monthly) variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * reanalysis-era5-single-levels-monthly-means -* reanalysis-era5-single-levels-monthly-means-preliminary-back-extension Pressure levels variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * reanalysis-era5-pressure-levels * reanalysis-era5-pressure-levels-monthly-means -* reanalysis-era5-pressure-levels-preliminary-back-extension -* reanalysis-era5-pressure-levels-monthly-means-preliminary-back-extension ERA5-land variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -251,7 +247,6 @@ "northward_turbulent_surface_stress", "ocean_surface_stress_equivalent_10m_neutral_wind_direction", "ocean_surface_stress_equivalent_10m_neutral_wind_speed", - "orography", # deprecated "peak_wave_period", "period_corresponding_to_maximum_individual_wave_height", "potential_evaporation", diff --git a/era5cli/key_management.py b/era5cli/key_management.py index a030e71..5b8d037 100644 --- a/era5cli/key_management.py +++ b/era5cli/key_management.py @@ -2,16 +2,15 @@ import sys from pathlib import Path from typing import Tuple -import cdsapi +import cads_api_client from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin ERA5CLI_CONFIG_PATH = Path.home() / ".config" / "era5cli" / "cds_key.txt" CDSAPI_CONFIG_PATH = Path.home() / ".cdsapirc" -DEFAULT_CDS_URL = "https://cds.climate.copernicus.eu/api/v2" +DEFAULT_CDS_URL = "https://cds-beta.climate.copernicus.eu/api" -AUTH_ERR_MSG = "401 Authorization Required" -NO_DATA_ERR_MSG = "There is no data matching your request" +AUTH_ERR_MSG = "401 Client Error" class InvalidRequestError(Exception): @@ -22,46 +21,22 @@ class InvalidLoginError(Exception): "Raised when an invalid login is provided to the cds server." -def attempt_cds_login(url: str, fullkey: str) -> True: +def attempt_cds_login(url: str, key: str) -> True: """Attempt to connect to the CDS, to validate the URL and UID + key. Args: url: URL to the CDS API. - fullkey: Combination of your UID and key, separated with a colon. + key: Combination of your UID and key, separated with a colon. Raises: ConnectionError: If no connection to the CDS could be made. InvalidLoginError: If an invalid authetication was provided to the CDS. InvalidRequestError: If the test request failed, likely due to changes in the CDS API's variable naming. - - Returns: - True if a connection was made succesfully. """ - connection = cdsapi.Client( - url=url, - key=fullkey, - verify=True, - quiet=True, # Supress output to the console from the test retrieve. - ) - + client = cads_api_client.ApiClient(key, url) try: - # Check the URL - connection.status() # pragma: no cover - - # Checks if the authentication works, without downloading data - connection.retrieve( # pragma: no cover - "reanalysis-era5-single-levels", - { - "variable": "2t", - "product_type": "reanalysis", - "date": "2012-12-01", - "time": "14:00", - "format": "netcdf", - }, - ) - return True - + client.check_authentication() except ConnectionError as err: raise ConnectionError( f"{os.linesep}Failed to connect to CDS. Please check your internet " @@ -74,37 +49,28 @@ def attempt_cds_login(url: str, fullkey: str) -> True: if AUTH_ERR_MSG in str(err): raise InvalidLoginError( f"{os.linesep}Authorization with the CDS served failed. Likely due to" - " an incorrect key or UID." + " an incorrect key." f"{os.linesep}Please check your era5cli configuration file: " f"{ERA5CLI_CONFIG_PATH.resolve()}{os.linesep}" "Or redefine your configuration with 'era5cli config'" ) from err - if NO_DATA_ERR_MSG in str(err): - raise InvalidRequestError( - f"{os.linesep}Something changed in the CDS API. Please raise an issue " - "on https://www.github.com/eWaterCycle/era5cli" - ) from err raise err # pragma: no cover def set_config( url: str, - uid: str, key: str, ) -> True: """Check the user-input configuration. Entry point for the CLI.""" try: - attempt_cds_login(url, fullkey=f"{uid}:{key}") - write_era5cli_config(url, uid, key) + attempt_cds_login(url, key) + write_era5cli_config(url, key) print( f"Keys succesfully validated and stored in {ERA5CLI_CONFIG_PATH.resolve()}" ) return True except InvalidLoginError: - print( - "Error: the UID and key are rejected by the CDS. " - "Please check and try again." - ) + print("Error: the key is rejected by the CDS. " "Please check and try again.") return False @@ -134,17 +100,15 @@ def valid_cdsapi_config() -> bool: True if a valid key has been found & written to file. Otherwise False. """ if CDSAPI_CONFIG_PATH.exists(): - url, fullkey = load_cdsapi_config() + url, key = load_cdsapi_config() try: - if sys.stdin.isatty() and attempt_cds_login(url, fullkey): + if sys.stdin.isatty() and attempt_cds_login(url, key): userinput = input( "Valid CDS keys found in the .cdsapirc file. Do you want to use " "these for era5cli? [Y/n]" ) if userinput.lower() in ["y", "yes", ""]: - set_config( - url, uid=fullkey.split(":")[0], key=fullkey.split(":")[1] - ) + set_config(url, key) return True except (ConnectionError, InvalidLoginError, InvalidRequestError): return False @@ -152,23 +116,39 @@ def valid_cdsapi_config() -> bool: def load_era5cli_config() -> Tuple[str, str]: + with open(ERA5CLI_CONFIG_PATH, encoding="utf8") as f: + contents = "".join(f.readlines()) + if "uid:" in contents: + msg = ( + "Old config detected. In the new CDS API only a key is required.\n" + "Please look at the new CDS website, and reconfigure your login in " + "era5cli\n" + " https://cds-beta.climate.copernicus.eu/" + ) + raise InvalidLoginError(msg) + with open(ERA5CLI_CONFIG_PATH, encoding="utf8") as f: url = f.readline().replace("url:", "").strip() - uid = f.readline().replace("uid:", "").strip() key = f.readline().replace("key:", "").strip() - return url, f"{uid}:{key}" + return url, key -def write_era5cli_config(url: str, uid: str, key: str): +def write_era5cli_config(url: str, key: str): ERA5CLI_CONFIG_PATH.parent.mkdir(exist_ok=True, parents=True) with open(ERA5CLI_CONFIG_PATH, mode="w", encoding="utf-8") as f: f.write(f"url: {url}\n") - f.write(f"uid: {uid}\n") f.write(f"key: {key}\n") def load_cdsapi_config() -> Tuple[str, str]: with open(CDSAPI_CONFIG_PATH, encoding="utf-8") as f: url = f.readline().replace("url:", "").strip() - fullkey = f.readline().replace("key:", "").strip() - return url, fullkey + key = f.readline().replace("key:", "").strip() + if ":" in key or "api/v2" in url: + msg = ( + "Your CDS API configuration file contains a UID entry/incorrect URL.\n" + "Please look at the new CDS website, and reconfigure your key:\n" + " https://cds-beta.climate.copernicus.eu/" + ) + raise InvalidLoginError(msg) + return url, key diff --git a/pyproject.toml b/pyproject.toml index 50e618f..602adc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,8 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "cdsapi == 0.5.1", + "cdsapi>=0.7.1", + "cads-api-client>=1.3", "pathos", "PTable", "netCDF4" diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d8257e..8a8785d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -269,23 +269,6 @@ def test_main_fetch(fetch): with pytest.raises(AssertionError): assert cli._execute(args) - # should give an AssertionError if years are out of bounds - argv = [ - "hourly", - "--startyear", - "1950", - "--variables", - "total_precipitation", - "--statistics", - "--endyear", - "2007", - "--ensemble", - "--prelimbe", - ] - args = cli._parse_args(argv) - with pytest.raises(AssertionError): - assert cli._execute(args) - # monthly call without endyear argv = [ "monthly", @@ -316,8 +299,6 @@ def test_main_info(info): config_args = [ "config", - "--uid", - "123456", "--key", "abc-def", ] @@ -325,7 +306,6 @@ def test_main_info(info): def test_config_parse(): args = cli._parse_args(config_args) - assert args.uid == "123456" assert args.key == "abc-def" @@ -347,7 +327,7 @@ def test_config_invalid(mock_a, mock_b, capfd): args = cli._parse_args(config_args) cli._execute(args) out, _ = capfd.readouterr() - assert "Error: the UID and key are rejected" in out + assert "Error: the key is rejected by the CDS." in out class TestConfigControlFlow: @@ -355,7 +335,7 @@ class TestConfigControlFlow: @mock.patch( "era5cli.key_management.load_era5cli_config", - return_value=("https://www.test.org/", "123:abc-def"), + return_value=("https://www.test.org/", "abc-def"), ) def test_config_show(self, mock, capsys): args = cli._parse_args(["config", "--show"]) @@ -363,7 +343,6 @@ def test_config_show(self, mock, capsys): expected = ( "Contents of .config/era5cli.txt:\n" - " uid: 123\n" " key: abc-def\n" " url: https://www.test.org/\n" ) @@ -373,11 +352,7 @@ def test_config_show(self, mock, capsys): @pytest.mark.parametrize( "input_args", [ - ["config", "--show", "--uid", "123"], ["config", "--show", "--key", "abc-def"], - ["config", "--show", "--uid", "123", "--key", "abc-def"], - ["config", "--key", "abc-def"], - ["config", "--uid", "123"], ], ) def test_config_inputerror(self, input_args): diff --git a/tests/test_config.py b/tests/test_config.py index f5c8026..29bd27d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,6 +4,9 @@ from era5cli import key_management +CFG_FILE = "url: https://www.github.com/\nkey: abc-def\n" + + @pytest.fixture(scope="function") def empty_path_era5(tmp_path_factory): return tmp_path_factory.mktemp("usrhome") / ".config" / "era5cli" / "cds_keys.txt" @@ -14,7 +17,7 @@ def valid_path_era5(tmp_path_factory): fn = tmp_path_factory.mktemp(".config") / "era5cli" / "cds_keys.txt" fn.parent.mkdir(parents=True) with open(fn, mode="w", encoding="utf-8") as f: - f.write("url: b\nuid: 123\nkey: abc-def\n") + f.write(CFG_FILE) return fn @@ -27,7 +30,7 @@ def empty_path_cds(tmp_path_factory): def valid_path_cds(tmp_path_factory): fn = tmp_path_factory.mktemp(".config") / "cdsapirc.txt" with open(fn, mode="w", encoding="utf-8") as f: - f.write("url: a\nkey: 123:abc-def") + f.write(CFG_FILE) return fn @@ -36,12 +39,15 @@ class TestEra5CliConfig: def test_set_config(self, empty_path_era5): with patch("era5cli.key_management.ERA5CLI_CONFIG_PATH", empty_path_era5): - key_management.write_era5cli_config(url="b", uid="123", key="abc-def") - assert key_management.load_era5cli_config() == ("b", "123:abc-def") + key_management.write_era5cli_config(url="b", key="abc-def") + assert key_management.load_era5cli_config() == ("b", "abc-def") def test_load_era5cli_config(self, valid_path_era5): with patch("era5cli.key_management.ERA5CLI_CONFIG_PATH", valid_path_era5): - assert key_management.load_era5cli_config() == ("b", "123:abc-def") + assert key_management.load_era5cli_config() == ( + "https://www.github.com/", + "abc-def", + ) def test_check_era5cli_config(self, valid_path_era5): mp1 = patch("era5cli.key_management.ERA5CLI_CONFIG_PATH", valid_path_era5) @@ -49,6 +55,12 @@ def test_check_era5cli_config(self, valid_path_era5): with mp1, mp2: key_management.check_era5cli_config() + def test_old_config(self, empty_path_era5): + with patch("era5cli.key_management.ERA5CLI_CONFIG_PATH", empty_path_era5): + key_management.write_era5cli_config(url="b", key="uid:abc-def") + with pytest.raises(key_management.InvalidLoginError, match="Old config"): + key_management.load_era5cli_config() + class TestConfigCdsrc: """Test the cases where a .cdsapirc file exists. @@ -82,7 +94,10 @@ def test_cdsrcfile_user_says_yes(self, empty_path_era5, valid_path_cds): with mp1, mp2, mp3, mp4, mp5: key_management.check_era5cli_config() with open(empty_path_era5, "r", encoding="utf-8") as f: - assert f.readlines() == ["url: a\n", "uid: 123\n", "key: abc-def\n"] + assert f.readlines() == [ + "url: https://www.github.com/\n", + "key: abc-def\n", + ] def test_cdsrcfile_invalid_keys(self, empty_path_era5, valid_path_cds): """.cdsapirc exists. url+key is validated, and is bad.""" @@ -107,41 +122,29 @@ class TestAttemptCdsLogin: expected. """ + @pytest.mark.xfail(reason="broken by new cads-client api") def test_status_fail(self): - with patch("cdsapi.Client.status", side_effect=rex.ConnectionError): + with patch( + "cads_api_client.ApiClient.check_authentication", + side_effect=rex.ConnectionError, + ): with pytest.raises(rex.ConnectionError, match="Failed to connect to CDS"): - key_management.attempt_cds_login(url="test", fullkey="abc:def") + key_management.attempt_cds_login( + url="https://www.github.com/", key="def" + ) def test_connection_fail(self): - mp1 = patch("cdsapi.Client.status") - mp2 = patch( - "cdsapi.Client.retrieve", - side_effect=Exception("401 Authorization Required"), + mp = patch( + "cads_api_client.ApiClient.check_authentication", + side_effect=Exception("401 Client Error"), ) - with mp1, mp2: + with mp: with pytest.raises( key_management.InvalidLoginError, match="Authorization with the CDS served failed", ): - key_management.attempt_cds_login(url="test", fullkey="abc:def") - - def test_retrieve_fail(self): - mp1 = patch("cdsapi.Client.status") - mp2 = patch( - "cdsapi.Client.retrieve", - side_effect=Exception("There is no data matching your request"), - ) - with mp1, mp2: - with pytest.raises( - key_management.InvalidRequestError, - match="Something changed in the CDS API", - ): - key_management.attempt_cds_login(url="test", fullkey="abc:def") + key_management.attempt_cds_login(url="test", key="abc:def") def test_all_pass(self): - mp1 = patch("cdsapi.Client.status") - mp2 = patch("cdsapi.Client.retrieve") - with mp1, mp2: - assert ( - key_management.attempt_cds_login(url="test", fullkey="abc:def") is True - ) + with patch("cads_api_client.ApiClient.check_authentication"): + key_management.attempt_cds_login(url="test", key="abc:def") diff --git a/tests/test_fetch.py b/tests/test_fetch.py index aa2608a..04ec51c 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -47,9 +47,8 @@ def initialize( months=list(range(1, 13)), days=list(range(1, 32)), hours=list(range(24)), - prelimbe=False, land=False, - splitmonths=False, + splitmonths=True, overwrite=False, ): with mock.patch( @@ -73,7 +72,6 @@ def initialize( pressurelevels=pressurelevels, merge=merge, threads=threads, - prelimbe=prelimbe, land=land, splitmonths=splitmonths, overwrite=overwrite, @@ -100,7 +98,6 @@ def test_init(mockpatch): pressurelevels="surface", merge=False, threads=2, - prelimbe=False, ) assert era5.months == ALL_MONTHS @@ -117,7 +114,6 @@ def test_init(mockpatch): assert era5.pressure_levels == "surface" assert not era5.merge assert era5.threads == 2 - assert not era5.prelimbe assert not era5.land # initializing hourly variable with days=None should result in ValueError @@ -137,9 +133,6 @@ def test_init(mockpatch): land=True, variables=["skin_temperature"], ensemble=False, splitmonths=False ) - with pytest.raises(ValueError, match="are not compatible"): - initialize(merge=True, splitmonths=True) - @mock.patch("cdsapi.Client", autospec=True) @mock.patch("era5cli.utils.append_history", autospec=True) @@ -289,9 +282,9 @@ def test_define_outputfilename(): "variables, years, merge, ensemble, splitmonths, expected", [ (_vars, _years, False, False, False, 2 * 3), - (_vars, _years, True, False, False, 2), # Test merge - (_vars, _years, False, True, False, 2 * 3), # Current default - (_vars, _years, False, True, True, 2 * 3 * 12), # Future default + (_vars, _years, True, False, False, 2), # test merge + (_vars, _years, False, True, False, 2 * 3), # old default + (_vars, _years, False, True, True, 2 * 3 * 12), # future default (_vars, _years[:1], False, False, False, 2 * 1), (_vars[:1], _years, False, False, False, 1 * 3), ], @@ -348,26 +341,9 @@ def test_product_type(): producttype = era5._product_type() assert producttype == "monthly_averaged_ensemble_members" - # Preliminary back extension monthly data have different names - era5.prelimbe = True - producttype = era5._product_type() - assert producttype == "members-monthly-means-of-daily-means" - - era5.synoptic = True - producttype = era5._product_type() - assert producttype == "members-synoptic-monthly-means" - - era5.ensemble = False - producttype = era5._product_type() - assert producttype == "reanalysis-synoptic-monthly-means" - - era5.synoptic = False - producttype = era5._product_type() - assert producttype == "reanalysis-monthly-means-of-daily-means" - # ERA5 land has more limited options era5.land = True - era5.prelimbe = False + era5.ensemble = False producttype = era5._product_type() assert producttype == "monthly_averaged_reanalysis" @@ -455,27 +431,8 @@ def test_build_name(): name = era5._build_name("total_precipitation")[0] assert name == "reanalysis-era5-single-levels-monthly-means" - # Test names for back extension - era5.prelimbe = True - name = era5._build_name("temperature")[0] - assert name == ( - "reanalysis-era5-pressure-levels-monthly-means" "-preliminary-back-extension" - ) - - name = era5._build_name("total_precipitation")[0] - assert name == ( - "reanalysis-era5-single-levels-monthly-means" "-preliminary-back-extension" - ) - - era5.period = "hourly" - name = era5._build_name("temperature")[0] - assert name == "reanalysis-era5-pressure-levels-preliminary-back-extension" - - name = era5._build_name("total_precipitation")[0] - assert name == "reanalysis-era5-single-levels-preliminary-back-extension" - # Tests for era5 land - era5.prelimbe = False + era5.period = "hourly" with pytest.raises(ValueError): era5._build_name("snow_cover") @@ -487,12 +444,6 @@ def test_build_name(): name = era5._build_name("snow_cover")[0] assert name == "reanalysis-era5-land-monthly-means" - # Test to interpret deprecated orography variable - era5 = initialize() - name, variable = era5._build_name("orography") - assert name == "reanalysis-era5-single-levels" - assert variable == "geopotential" - era5 = initialize() name = era5._build_name("geopotential")[0] assert name == "reanalysis-era5-pressure-levels" @@ -506,7 +457,12 @@ def test_build_name(): def test_build_request(): """Test _build_request function of Fetch class.""" # hourly data - era5 = initialize(period="hourly", variables=["total_precipitation"], years=[2008]) + era5 = initialize( + period="hourly", + variables=["total_precipitation"], + years=[2008], + splitmonths=False, + ) (name, request) = era5._build_request("total_precipitation", [2008]) assert name == "reanalysis-era5-single-levels" req = { @@ -534,24 +490,6 @@ def test_build_request(): } assert request == req - # preliminary back extension - era5 = initialize( - period="monthly", variables=["total_precipitation"], years=[1970], prelimbe=True - ) - (name, request) = era5._build_request("total_precipitation", [1970]) - assert name == ( - "reanalysis-era5-single-levels-monthly" "-means-preliminary-back-extension" - ) - req = { - "variable": "total_precipitation", - "year": [1970], - "product_type": "members-monthly-means-of-daily-means", - "month": ALL_MONTHS, - "time": ALL_HOURS, - "format": "netcdf", - } - assert request == req - # land era5 = initialize( period="monthly", variables=["snow_cover"], hours=[0], land=True, ensemble=False @@ -573,18 +511,9 @@ def test_build_request(): with pytest.raises(ValueError): era5 = initialize(variables=["temperature"], pressurelevels=None) - # requesting data from orography should call geopotential - era5 = initialize() - (name, request) = era5._build_request("orography", [2008]) - assert request["variable"] == "geopotential" - def test_incompatible_options(): """Test that invalid combinations of arguments don't silently pass.""" - era5 = initialize(land=True, prelimbe=True) - with pytest.raises(ValueError): - era5._build_request("total_precipitation", [2008]) - era5 = initialize(land=False) with pytest.raises(ValueError): era5._build_request("snow_cover", [2008]) diff --git a/tests/test_info.py b/tests/test_info.py index 7aefa53..e25f8f3 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,4 +1,5 @@ """Tests for era5cli Fetch class.""" + import pytest from era5cli import info diff --git a/tests/test_integration.py b/tests/test_integration.py index 859fa77..cad4081 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -17,35 +17,12 @@ def my_thing_mock(): # combine calls with result and possible warning message call_result = [ - { - # orography is translated to geopotential in the query - "call": dedent( - """\ - era5cli hourly --variables orography --startyear 2008 --dryrun - """ - ), - "result": dedent( - """\ - reanalysis-era5-single-levels {'variable': 'geopotential', 'year': - 2008, 'month': ['01', '02', '03', '04', '05', '06', '07', '08', - '09', '10', '11', '12'], 'time': ['00:00', '01:00', '02:00', - '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', - '10:00', '11:00', '12:00', '13:00', '14:00', '15:00', '16:00', - '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00'], - 'format': 'netcdf', 'product_type': 'reanalysis', 'day': ['01', - '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', - '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', - '24', '25', '26', '27', '28', '29', '30', '31']} - era5_orography_2008_hourly.nc""" - ), - "warn": "The variable 'orography' has been deprecated by CDS.", - }, { # geopotential needs '--levels surface' to be correctly interpreted "call": dedent( """\ era5cli hourly --variables geopotential --startyear 2008 --dryrun - --levels surface""" + --splitmonths False --levels surface""" ), "result": dedent( """\ @@ -74,7 +51,7 @@ def my_thing_mock(): "result": dedent( """\ reanalysis-era5-pressure-levels {'variable': 'geopotential', - 'year': 2008, 'month': ['01'], 'time': ['00:00', '01:00', '02:00', + 'year': 2008, 'month': '01', 'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', '10:00', '11:00', '12:00', '13:00', '14:00', '15:00', '16:00', '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00'], @@ -85,31 +62,10 @@ def my_thing_mock(): '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31']} - era5_geopotential_2008_hourly.nc""" + era5_geopotential_2008-01_hourly.nc""" ), "warn": "Getting variable from pressure level data.", }, - { - # preliminary-back-extension is combined with monthly-means - "call": dedent( - """\ - era5cli monthly --variables temperature --startyear 1960 --prelimbe - --dryrun""" - ), - "result": dedent( - """\ - reanalysis-era5-pressure-levels-monthly-means-preliminary-back-extension - {'variable': 'temperature', 'year': 1960, 'month': ['01', '02', - '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'], - 'time': ['00:00'], 'format': 'netcdf', 'pressure_level': [1, 2, 3, - 5, 7, 10, 20, 30, 50, 70, 100, 125, 150, 175, 200, 225, 250, 300, - 350, 400, 450, 500, 550, 600, 650, 700, 750, 775, 800, 825, 850, - 875, 900, 925, 950, 975, 1000], 'product_type': - 'reanalysis-monthly-means-of-daily-means'} - era5_temperature_1960_monthly.nc""" - ), - "warn": "The years of the ERA5 preliminary back extension", - }, { # era5-Land is combined with monthly means "call": dedent( @@ -148,7 +104,7 @@ def test_main(call_result, capsys, caplog): with caplog.at_level(logging.INFO): with mock.patch( "era5cli.fetch.key_management.load_era5cli_config", - return_value=("url", "key:uid"), + return_value=("url", "key"), ): main(call) captured = capsys.readouterr().out