Move to new CDS, prep for 2.0.0b1 release

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.2
+current_version = 2.0.0b1
 
 [comment]
 comment = The contents of this file cannot be merged with that of pyproject.toml until https://github.com/c4urself/bump2version/issues/42 is resolved

.github/.codecov.yml

@@ -1,19 +1,15 @@
-#
-# This codecov.yml is the default configuration for
-# all repositories on Codecov. You may adjust the settings
-# below in your own codecov.yml in your repository.
-#
+comment:
+  behavior: default
+  layout: header, diff
 coverage:
   precision: 2
+  range:
+  - 70.0
+  - 100.0
   round: down
-  range: 70...100
   status:
-    # Learn more at https://codecov.io/docs#yaml_default_commit_status
-    project: true
-    patch: true
     changes: false
+    patch: true
+    project: true
 ignore:
-  - "tests"
-comment:
-  layout: header, diff
-  behavior: default  # update if exists else create new
+- ^tests.*

.github/workflows/test_codecov.yml

@@ -22,10 +22,11 @@ jobs:
     - name: Generate test coverage
       run: hatch run coverage
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         directory: ./coverage/reports/
         files: ./cov.xml
         env_vars: python-version
         fail_ci_if_error: true
         verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }} # required

CITATION.cff

@@ -97,4 +97,4 @@ license: Apache-2.0
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ewatercycle/era5cli"
 title: era5cli
-version: "1.4.2"
+version: "2.0.0b1"

README.md

@@ -1,5 +1,21 @@
 <img align="right" width="150" alt="Logo" src="docs/assets/era5cli_logo_colors_border.png">
 
+> [!IMPORTANT]
+> The old Climate Data Store (CDS) will be shut down on 3 September 2024.
+> All era5cli versions up to v1.4.2 will no longer work.
+>
+> For more information see: https://forum.ecmwf.int/t/the-new-climate-data-store-beta-cds-beta-is-now-live/3315
+>
+> 
+> To continue using era5cli, you will need to re-register at ECMWF and get a new API key,
+> and transition to the era5cli v2 beta. This can be installed with:
+> `pip install era5cli==2.0.0b1` 
+
+> [!WARNING]
+> netCDF files from the new Climate Data Store Beta are not formatted the same as the
+> old CDS. Some variables might be missing.
+> See the open issue [here](https://github.com/eWaterCycle/era5cli/issues/165), as well as the [ECMWF discussion forum](https://forum.ecmwf.int/).
+
 
 [![github license badge](https://img.shields.io/github/license/eWaterCycle/era5cli)](https://github.com/eWaterCycle/era5cli)
 [![rsd badge](https://img.shields.io/badge/RSD-era5cli-blue)](https://research-software-directory.org/software/era5cli)

docs/CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 2.0.0b1 - 2024-08-25
+
+**Fixed:**
+
+ - Added support for the beta-CDS
+
 ## 1.4.2 - 2023-12-12
 
 **Fixed:**

docs/getting_started.md

@@ -43,10 +43,10 @@ After activating your account, **login** on the CDS website, go to your **profil
 To configure era5cli to use these keys, open up the environment you installed era5cli in, and do:
 
 ```sh
-era5cli config --uid ID_NUMBER --key "KEY"
+era5cli config --key "KEY"
 ```
 
-*Where ID_NUMBER is your user ID (e.g. 123456) and "KEY" is your API key, inside double quotes (e.g. "4s215sgs-2dfa-6h34-62h2-1615ad163414").*
+*Where "KEY" is your API key, inside double quotes (e.g. "4s215sgs-2dfa-6h34-62h2-1615ad163414").*
 
 After running this command your ID and key are validated and stored inside your home folder, under `.config/era5cli/cds_key.txt`.
 

docs/index.md

@@ -2,6 +2,22 @@
 
 A command line interface to download ERA5 data from the [Copernicus Climate Data Store](https://climate.copernicus.eu/).
 
+???+ note
+    The old Climate Data Store (CDS) will be shut down on 3 September 2024.
+    All era5cli versions up to v1.4.2 will no longer work.
+
+    For more information see: https://forum.ecmwf.int/t/the-new-climate-data-store-beta-cds-beta-is-now-live/3315
+
+    To continue using era5cli, you will need to re-register at ECMWF and get a new API key,
+    and transition to the era5cli v2 beta. This can be installed with:
+    `pip install era5cli==2.0.0b1` 
+
+???+ warning
+    netCDF files from the new Climate Data Store Beta are not formatted the same as the
+    old CDS. Some variables might be missing.
+
+    See the open issue [here](https://github.com/eWaterCycle/era5cli/issues/165), as well as the [ECMWF discussion forum](https://forum.ecmwf.int/).
+
 With era5cli you can:
 
  - Download meteorological data in GRIB/NetCDF, including ERA5 data from the preliminary back extension, and ERA5-Land data.

era5cli/__version__.py

@@ -26,4 +26,4 @@
     "Bart Schilperoort",
 )
 __email__ = "[email protected]"
-__version__ = "1.4.2"
+__version__ = "2.0.0b1"

era5cli/args/config.py

@@ -10,7 +10,6 @@
 
     Adds the 'config' parser with the following arguments:
         --show
-        --uid
         --key
         --url
 
@@ -27,7 +26,7 @@
             This will create a config file in your home directory, in folder named
             ".config". The CDS URL, your UID and the CDS keys will be stored here.
 
-            To find your key and UID, go to https://cds.climate.copernicus.eu/ and
+            To find your key, go to https://beta-cds.climate.copernicus.eu/ and
             login with your email and password. Then go to your user profile (top
             right).
 
@@ -54,16 +53,6 @@
         ),
     )
 
-    config.add_argument(
-        "--uid",
-        type=str,
-        help=textwrap.dedent(
-            """
-            Your CDS User ID, e.g.: 123456
-            """
-        ),
-    )
-
     config.add_argument(
         "--key",
         type=str,
@@ -87,6 +76,18 @@
         ),
     )
 
+    config.add_argument(
+        "--uid",
+        type=str,
+        required=False,
+        default="",
+        help=textwrap.dedent(
+            """
+            DO NOT USE: deprecated due to changes in the CDS API"
+            """
+        ),
+    )
+
 
 class InputError(Exception):
     "Raised when a user inputs an invalid combination of arguments."
@@ -101,27 +102,22 @@
 
     Args:
         args: Arguments collected by argparse
-
-    Returns:
-        True
     """
-    if args.show and any((args.uid, args.key)):
+    if len(args.uid) > 0:
+        msg = (
+            "The `uid` argument is deprecated.\n"
+            "The new CDS API does not use UIDs anymore."
+        )
+        raise InputError(msg)
+
+    if args.show and args.key is not None:
         raise InputError("Either call `show` or set the key. Not both.")
-    if not args.show and (args.uid is None or args.key is None):
-        raise InputError("Both the UID and the key are required inputs.")
+    if not args.show and args.key is None:
+        raise InputError("Your CDS API key is a required input.")
     if args.show:
-        url, fullkey = key_management.load_era5cli_config()
-        uid, key = fullkey.split(":")
+        url, key = key_management.load_era5cli_config()
         print(
-            "Contents of .config/era5cli.txt:\n"
-            f"    uid: {uid}\n"
-            f"    key: {key}\n"
-            f"    url: {url}\n"
+            "Contents of .config/era5cli.txt:\n" f"    key: {key}\n" f"    url: {url}\n"
         )
-        return True
-
-    return key_management.set_config(
-        url=args.url,
-        uid=args.uid,
-        key=args.key,
-    )
+    else:
+        key_management.set_config(args.url, args.key)

era5cli/key_management.py

@@ -8,7 +8,7 @@
 
 ERA5CLI_CONFIG_PATH = Path.home() / ".config" / "era5cli" / "cds_key.txt"
 CDSAPI_CONFIG_PATH = Path.home() / ".cdsapirc"
-DEFAULT_CDS_URL = "https://cds.climate.copernicus.eu/api/v2"
+DEFAULT_CDS_URL = "https://cds-beta.climate.copernicus.eu/api"
 
 AUTH_ERR_MSG = "401 Authorization Required"
 NO_DATA_ERR_MSG = "There is no data matching your request"
@@ -22,12 +22,12 @@
     "Raised when an invalid login is provided to the cds server."
 
 
-def attempt_cds_login(url: str, fullkey: str) -> True:
+def attempt_cds_login(url: str, key: str) -> True:
     """Attempt to connect to the CDS, to validate the URL and UID + key.
 
     Args:
         url: URL to the CDS API.
-        fullkey: Combination of your UID and key, separated with a colon.
+        key: Combination of your UID and key, separated with a colon.
 
     Raises:
         ConnectionError: If no connection to the CDS could be made.
@@ -40,14 +40,14 @@
     """
     connection = cdsapi.Client(
         url=url,
-        key=fullkey,
+        key=key,
         verify=True,
         quiet=True,  # Supress output to the console from the test retrieve.
     )
 
     try:
-        # Check the URL
-        connection.status()  # pragma: no cover
+        # Check the URL (broken in 0.7.0...)
+        # connection.status()
 
         # Checks if the authentication works, without downloading data
         connection.retrieve(  # pragma: no cover
@@ -74,7 +74,7 @@
         if AUTH_ERR_MSG in str(err):
             raise InvalidLoginError(
                 f"{os.linesep}Authorization with the CDS served failed. Likely due to"
-                " an incorrect key or UID."
+                " an incorrect key."
                 f"{os.linesep}Please check your era5cli configuration file: "
                 f"{ERA5CLI_CONFIG_PATH.resolve()}{os.linesep}"
                 "Or redefine your configuration with 'era5cli config'"
@@ -89,22 +89,18 @@
 
 def set_config(
     url: str,
-    uid: str,
     key: str,
 ) -> True:
     """Check the user-input configuration. Entry point for the CLI."""
     try:
-        attempt_cds_login(url, fullkey=f"{uid}:{key}")
-        write_era5cli_config(url, uid, key)
+        attempt_cds_login(url, key)
+        write_era5cli_config(url, key)
         print(
             f"Keys succesfully validated and stored in {ERA5CLI_CONFIG_PATH.resolve()}"
         )
         return True
     except InvalidLoginError:
-        print(
-            "Error: the UID and key are rejected by the CDS. "
-            "Please check and try again."
-        )
+        print("Error: the key is rejected by the CDS. " "Please check and try again.")
     return False
 
 
@@ -134,41 +130,55 @@
         True if a valid key has been found & written to file. Otherwise False.
     """
     if CDSAPI_CONFIG_PATH.exists():
-        url, fullkey = load_cdsapi_config()
+        url, key = load_cdsapi_config()
         try:
-            if sys.stdin.isatty() and attempt_cds_login(url, fullkey):
+            if sys.stdin.isatty() and attempt_cds_login(url, key):
                 userinput = input(
                     "Valid CDS keys found in the .cdsapirc file. Do you want to use "
                     "these for era5cli? [Y/n]"
                 )
                 if userinput.lower() in ["y", "yes", ""]:
-                    set_config(
-                        url, uid=fullkey.split(":")[0], key=fullkey.split(":")[1]
-                    )
+                    set_config(url, key)
                     return True
         except (ConnectionError, InvalidLoginError, InvalidRequestError):
             return False
     return False
 
 
 def load_era5cli_config() -> Tuple[str, str]:
+    with open(ERA5CLI_CONFIG_PATH, encoding="utf8") as f:
+        contents = "".join(f.readlines())
+        if "uid" in contents:
+            msg = (
+                "Old config detected. In the new CDS API only a key is required.\n"
+                "Please look at the new CDS website, and reconfigure your login in "
+                "era5cli\n"
+                "    https://cds-beta.climate.copernicus.eu/"
+            )
+            raise InvalidLoginError(msg)
+
     with open(ERA5CLI_CONFIG_PATH, encoding="utf8") as f:
         url = f.readline().replace("url:", "").strip()
-        uid = f.readline().replace("uid:", "").strip()
         key = f.readline().replace("key:", "").strip()
-    return url, f"{uid}:{key}"
+    return url, key
 
 
-def write_era5cli_config(url: str, uid: str, key: str):
+def write_era5cli_config(url: str, key: str):
     ERA5CLI_CONFIG_PATH.parent.mkdir(exist_ok=True, parents=True)
     with open(ERA5CLI_CONFIG_PATH, mode="w", encoding="utf-8") as f:
         f.write(f"url: {url}\n")
-        f.write(f"uid: {uid}\n")
         f.write(f"key: {key}\n")
 
 
 def load_cdsapi_config() -> Tuple[str, str]:
     with open(CDSAPI_CONFIG_PATH, encoding="utf-8") as f:
         url = f.readline().replace("url:", "").strip()
-        fullkey = f.readline().replace("key:", "").strip()
-    return url, fullkey
+        key = f.readline().replace("key:", "").strip()
+        if ":" in key or "api/v2" in url:
+            msg = (
+                "Your CDS API configuration file contains a UID entry/incorrect URL.\n"
+                "Please look at the new CDS website, and reconfigure your key:\n"
+                "    https://cds-beta.climate.copernicus.eu/"
+            )
+            raise InvalidLoginError(msg)
+    return url, key

pyproject.toml

@@ -57,7 +57,8 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
-  "cdsapi == 0.5.1",
+  "cdsapi>=0.7.1",
+  "cads-api-client>=1.2",
   "pathos",
   "PTable",
   "netCDF4"