diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index 58aa806a7..06c3c0e4f 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -1,25 +1,25 @@ build: modules: - - ninja + - ninja dependencies: - - ecmwf/ecbuild@develop - - MathisRosenhauer/libaec@refs/tags/v1.1.3 - - ecmwf/eccodes@develop - - ecmwf/eckit@develop - - ecmwf/odc@develop - - ecmwf/metkit@develop - - ecmwf/fdb@develop + - ecmwf/ecbuild@develop + - MathisRosenhauer/libaec@refs/tags/v1.1.3 + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + - ecmwf/odc@develop + - ecmwf/metkit@develop + - ecmwf/fdb@develop python_dependencies: - - ecmwf/eccodes-python@develop - - ecmwf/cfgrib@master - - ecmwf/pdbufr@master - - ecmwf/pyodc@develop - - ecmwf/pyfdb@develop - - ecmwf/earthkit-utils@develop + - ecmwf/eccodes-python@develop + - ecmwf/cfgrib@master + - ecmwf/pdbufr@master + - ecmwf/pyodc@develop + - ecmwf/pyfdb@develop + - ecmwf/earthkit-utils@develop env: - - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples - - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions - - GITHUB_WORKFLOW=anything + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions + - GITHUB_WORKFLOW=anything parallel: 64 requirements: tests/downstream-ci-requirements.txt pytest_cmd: | diff --git a/.github/ci-hpc-gpu-config.yml b/.github/ci-hpc-gpu-config.yml index 0c0876c2a..8e1e99200 100644 --- a/.github/ci-hpc-gpu-config.yml +++ b/.github/ci-hpc-gpu-config.yml @@ -1,24 +1,24 @@ build: modules: - - ninja + - ninja dependencies: - - ecmwf/ecbuild@develop - - MathisRosenhauer/libaec@refs/tags/v1.1.3 - - ecmwf/eccodes@develop - - ecmwf/eckit@develop - - ecmwf/odc@develop - - ecmwf/metkit@develop - - ecmwf/fdb@develop + - ecmwf/ecbuild@develop + - MathisRosenhauer/libaec@refs/tags/v1.1.3 + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + - ecmwf/odc@develop + - ecmwf/metkit@develop + - ecmwf/fdb@develop python_dependencies: - - ecmwf/eccodes-python@develop - - ecmwf/cfgrib@master - - ecmwf/pdbufr@master - - ecmwf/pyodc@develop - - ecmwf/pyfdb@develop - - ecmwf/earthkit-utils@develop + - ecmwf/eccodes-python@develop + - ecmwf/cfgrib@master + - ecmwf/pdbufr@master + - ecmwf/pyodc@develop + - ecmwf/pyfdb@develop + - ecmwf/earthkit-utils@develop env: - - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples - - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions parallel: 64 queue: ng gpus: 1 diff --git a/.github/workflows/cd-pypi.yml b/.github/workflows/cd-pypi.yml index ae8a69c87..438ebc420 100644 --- a/.github/workflows/cd-pypi.yml +++ b/.github/workflows/cd-pypi.yml @@ -2,7 +2,7 @@ name: cd on: push: tags: - - '**' + - '**' jobs: deploy: uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f031df8f0..aea227fce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,26 +3,26 @@ on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - - "main" - - "develop" + - "main" + - "develop" tags-ignore: - - "**" + - "**" paths-ignore: - - "docs/**" - - "README.md" + - "docs/**" + - "README.md" # Trigger the workflow on pull request pull_request: paths-ignore: - - "docs/**" - - "README.md" + - "docs/**" + - "README.md" # Trigger the workflow manually workflow_dispatch: # Trigger after public PR approved for CI pull_request_target: types: [labeled] paths-ignore: - - "docs/**" - - "README.md" + - "docs/**" + - "README.md" jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: diff --git a/.github/workflows/nightly-hpc-gpu.yml b/.github/workflows/nightly-hpc-gpu.yml index cca64324f..03ebef932 100644 --- a/.github/workflows/nightly-hpc-gpu.yml +++ b/.github/workflows/nightly-hpc-gpu.yml @@ -3,16 +3,16 @@ on: workflow_dispatch: # Run at 04:00 UTC every day (on default branch) schedule: - - cron: "0 04 * * *" + - cron: "0 04 * * *" jobs: test-hpc-gpu: runs-on: [self-hosted, linux, hpc] steps: - - uses: ecmwf/reusable-workflows/ci-hpc@v2 - with: - github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} - github_token: ${{ secrets.GH_REPO_READ_TOKEN }} - troika_user: ${{ secrets.HPC_CI_TESTING_SSH_USER }} - repository: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }} - build_config: .github/ci-hpc-gpu-config.yml - python_version: "3.10" + - uses: ecmwf/reusable-workflows/ci-hpc@v2 + with: + github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }} + github_token: ${{ secrets.GH_REPO_READ_TOKEN }} + troika_user: ${{ secrets.HPC_CI_TESTING_SSH_USER }} + repository: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }} + build_config: .github/ci-hpc-gpu-config.yml + python_version: "3.10" diff --git a/.github/workflows/notify-new-issue.yml b/.github/workflows/notify-new-issue.yml index 3593edc42..4896baafe 100644 --- a/.github/workflows/notify-new-issue.yml +++ b/.github/workflows/notify-new-issue.yml @@ -2,12 +2,12 @@ name: Notify new issue on: issues: types: - - "opened" + - "opened" jobs: notify: runs-on: ubuntu-latest steps: - - name: Notify new issue - uses: ecmwf/notify-teams-issue@v1 - with: - incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} + - name: Notify new issue + uses: ecmwf/notify-teams-issue@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.github/workflows/notify-new-pr.yml b/.github/workflows/notify-new-pr.yml index af5dd82f7..2c22ebad6 100644 --- a/.github/workflows/notify-new-pr.yml +++ b/.github/workflows/notify-new-pr.yml @@ -2,12 +2,12 @@ name: Notify new PR on: pull_request_target: types: - - "opened" + - "opened" jobs: notify: runs-on: ubuntu-latest steps: - - name: Notify new PR - uses: ecmwf/notify-teams-pr@v1 - with: - incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} + - name: Notify new PR + uses: ecmwf/notify-teams-pr@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3363df3da..86aac409d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,54 +1,41 @@ repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: trailing-whitespace # Trailing whitespace checker - - id: end-of-file-fixer # Ensure files end in a newline - - id: check-json - - id: check-yaml # Check YAML files for syntax errors only - args: [--unsafe, --allow-multiple-documents] - - id: check-toml - # - id: check-added-large-files - - id: debug-statements # Check for debugger imports and py37+ breakpoint() - - id: mixed-line-ending - - id: no-commit-to-branch # Prevent committing to main / master - - id: check-merge-conflict # Check for files that contain merge conflict - exclude: /README\.rst$|^docs/.*\.rst$ - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black - args: [--line-length=120] - - repo: https://github.com/keewis/blackdoc - rev: v0.3.8 - hooks: - - id: blackdoc - additional_dependencies: [black==23.3.0] - exclude: xr_engine_profile_rst\.py - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 - hooks: - - id: ruff - exclude: '(docs/conf|dev/.*|.*_)\.py$|docs/experimental/.*\.ipynb$|polytope_feature\.ipynb$' - args: - - --line-length=120 - - --fix - - --exit-non-zero-on-fix - - --preview - - repo: https://github.com/executablebooks/mdformat - rev: 0.7.14 - hooks: - - id: mdformat - exclude: cruft-update-template.md - - repo: https://github.com/google/yamlfmt - rev: v0.13.0 - hooks: - - id: yamlfmt - - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 - hooks: - - id: sphinx-lint - - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.5.0" - hooks: - - id: pyproject-fmt +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace # Trailing whitespace checker + - id: end-of-file-fixer # Ensure files end in a newline + - id: check-json + - id: check-yaml # Check YAML files for syntax errors only + args: [--unsafe, --allow-multiple-documents] + - id: check-toml + # - id: check-added-large-files + - id: debug-statements # Check for debugger imports and py37+ breakpoint() + - id: mixed-line-ending + - id: no-commit-to-branch # Prevent committing to main / master + - id: check-merge-conflict # Check for files that contain merge conflict + exclude: /README\.rst$|^docs/.*\.rst$ +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.4 + hooks: + - id: ruff-check + exclude: '(dev/.*|.*_)\.py$' + args: + - --fix + - --exit-non-zero-on-fix + - id: ruff-format +- repo: https://github.com/executablebooks/mdformat + rev: 0.7.14 + hooks: + - id: mdformat + exclude: cruft-update-template.md +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.16.0 + hooks: + - id: pretty-format-yaml + args: [--autofix, --preserve-quotes] + - id: pretty-format-toml + args: [--autofix] +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint diff --git a/.readthedocs.yml b/.readthedocs.yml index 4d980e203..900dcb4af 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,8 +6,8 @@ build: python: "3.10" python: install: - - requirements: docs/requirements.txt - - method: pip - path: . + - requirements: docs/requirements.txt + - method: pip + path: . sphinx: configuration: docs/conf.py diff --git a/docs/examples/bufr_synop.ipynb b/docs/examples/bufr_synop.ipynb index 113f3f23d..54815588b 100644 --- a/docs/examples/bufr_synop.ipynb +++ b/docs/examples/bufr_synop.ipynb @@ -488,8 +488,7 @@ } ], "source": [ - "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \n", - " \"heightOfStation\", \"airTemperatureAt2M\"])\n", + "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \"heightOfStation\", \"airTemperatureAt2M\"])\n", "df" ] }, @@ -576,9 +575,10 @@ } ], "source": [ - "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \n", - " \"heightOfStation\", \"airTemperatureAt2M\", \"WMO_station_id\"],\n", - " filters={\"WMO_station_id\": [30846, 89514]})\n", + "df = ds.to_pandas(\n", + " columns=[\"latitude\", \"longitude\", \"heightOfStation\", \"airTemperatureAt2M\", \"WMO_station_id\"],\n", + " filters={\"WMO_station_id\": [30846, 89514]},\n", + ")\n", "df" ] }, @@ -662,9 +662,10 @@ } ], "source": [ - "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \n", - " \"heightOfStation\", \"airTemperatureAt2M\"],\n", - " filters={\"airTemperatureAt2M\": slice(None, 290)})\n", + "df = ds.to_pandas(\n", + " columns=[\"latitude\", \"longitude\", \"heightOfStation\", \"airTemperatureAt2M\"],\n", + " filters={\"airTemperatureAt2M\": slice(None, 290)},\n", + ")\n", "df" ] }, @@ -756,9 +757,10 @@ } ], "source": [ - "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \n", - " \"heightOfStation\", \"airTemperatureAt2M\"],\n", - " filters={\"airTemperatureAt2M\": slice(290, 300)})\n", + "df = ds.to_pandas(\n", + " columns=[\"latitude\", \"longitude\", \"heightOfStation\", \"airTemperatureAt2M\"],\n", + " filters={\"airTemperatureAt2M\": slice(290, 300)},\n", + ")\n", "df" ] }, @@ -842,9 +844,10 @@ } ], "source": [ - "df = ds.to_pandas(columns=[\"latitude\", \"longitude\", \n", - " \"heightOfStation\", \"airTemperatureAt2M\"],\n", - " filters={\"airTemperatureAt2M\": slice(300, None)})\n", + "df = ds.to_pandas(\n", + " columns=[\"latitude\", \"longitude\", \"heightOfStation\", \"airTemperatureAt2M\"],\n", + " filters={\"airTemperatureAt2M\": slice(300, None)},\n", + ")\n", "df" ] }, diff --git a/docs/examples/bufr_temp.ipynb b/docs/examples/bufr_temp.ipynb index 11031cbf2..907d6173e 100644 --- a/docs/examples/bufr_temp.ipynb +++ b/docs/examples/bufr_temp.ipynb @@ -1159,8 +1159,9 @@ } ], "source": [ - "df = ds.to_pandas(columns=(\"WMO_station_id\", \"data_datetime\", \"pressure\", \"airTemperature\"),\n", - " filters={\"WMO_station_id\": 1415})\n", + "df = ds.to_pandas(\n", + " columns=(\"WMO_station_id\", \"data_datetime\", \"pressure\", \"airTemperature\"), filters={\"WMO_station_id\": 1415}\n", + ")\n", "df" ] }, diff --git a/docs/examples/cache.ipynb b/docs/examples/cache.ipynb index 3a53ec1f7..f8d9269c2 100644 --- a/docs/examples/cache.ipynb +++ b/docs/examples/cache.ipynb @@ -21,8 +21,7 @@ "metadata": {}, "outputs": [], "source": [ - "from earthkit.data import cache\n", - "from earthkit.data import config" + "from earthkit.data import cache, config" ] }, { @@ -215,8 +214,7 @@ } ], "source": [ - "s = {\"cache-policy\": \"off\", \n", - " \"temporary-directory-root\": \"~/my_demo_tmp\"}\n", + "s = {\"cache-policy\": \"off\", \"temporary-directory-root\": \"~/my_demo_tmp\"}\n", "config.set(s)\n", "cache.directory()" ] @@ -358,8 +356,7 @@ } ], "source": [ - "s = {\"cache-policy\": \"temporary\", \n", - " \"temporary-cache-directory-root\": \"~/my_demo_cache\"}\n", + "s = {\"cache-policy\": \"temporary\", \"temporary-cache-directory-root\": \"~/my_demo_cache\"}\n", "config.set(s)\n", "cache.directory()" ] diff --git a/docs/examples/demo_sources_plugin.ipynb b/docs/examples/demo_sources_plugin.ipynb index 6026f9f04..e3daf2f12 100644 --- a/docs/examples/demo_sources_plugin.ipynb +++ b/docs/examples/demo_sources_plugin.ipynb @@ -188,11 +188,11 @@ ], "source": [ "ds = ekd.from_source(\n", - " \"demo-source\",\n", - " \"sqlite:///_test.db\",\n", - " \"select * from data;\",\n", - " parse_dates=[\"time\"],\n", - " )\n", + " \"demo-source\",\n", + " \"sqlite:///_test.db\",\n", + " \"select * from data;\",\n", + " parse_dates=[\"time\"],\n", + ")\n", "df = ds.to_pandas()\n", "df" ] diff --git a/docs/examples/ecmwf_open_data.ipynb b/docs/examples/ecmwf_open_data.ipynb index e35719fbd..9c3674783 100644 --- a/docs/examples/ecmwf_open_data.ipynb +++ b/docs/examples/ecmwf_open_data.ipynb @@ -69,11 +69,7 @@ "source": [ "ds = ekd.from_source(\n", " \"ecmwf-open-data\",\n", - " request=dict(\n", - " param=[\"t\", \"gh\"],\n", - " levelist=\"500\",\n", - " step=[0, 24, 48]\n", - " ),\n", + " request=dict(param=[\"t\", \"gh\"], levelist=\"500\", step=[0, 24, 48]),\n", ")" ] }, diff --git a/docs/examples/fdb.ipynb b/docs/examples/fdb.ipynb index 2c66db126..85cfef48c 100644 --- a/docs/examples/fdb.ipynb +++ b/docs/examples/fdb.ipynb @@ -53,16 +53,16 @@ "outputs": [], "source": [ "request = {\n", - " 'class': 'od',\n", - " 'expver': '0001',\n", - " 'stream': 'oper',\n", - " 'date': '20240421',\n", - " 'time': [0, 12],\n", - " 'domain': 'g',\n", - " 'type': 'an',\n", - " 'levtype': 'sfc',\n", - " 'step': 0,\n", - " 'param': [151, 167, 168]\n", + " \"class\": \"od\",\n", + " \"expver\": \"0001\",\n", + " \"stream\": \"oper\",\n", + " \"date\": \"20240421\",\n", + " \"time\": [0, 12],\n", + " \"domain\": \"g\",\n", + " \"type\": \"an\",\n", + " \"levtype\": \"sfc\",\n", + " \"step\": 0,\n", + " \"param\": [151, 167, 168],\n", "}" ] }, diff --git a/docs/examples/fields_from_dict_in_loop.ipynb b/docs/examples/fields_from_dict_in_loop.ipynb index 9bb809686..bf2790ff3 100644 --- a/docs/examples/fields_from_dict_in_loop.ipynb +++ b/docs/examples/fields_from_dict_in_loop.ipynb @@ -44,20 +44,20 @@ "outputs": [], "source": [ "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0],\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + "}\n", "\n", "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 600, **prototype},\n", - " ]" + " {\"param\": \"t\", \"level\": 500, **prototype},\n", + " {\"param\": \"t\", \"level\": 850, **prototype},\n", + " {\"param\": \"u\", \"level\": 500, **prototype},\n", + " {\"param\": \"u\", \"level\": 850, **prototype},\n", + " {\"param\": \"d\", \"level\": 850, **prototype},\n", + " {\"param\": \"d\", \"level\": 600, **prototype},\n", + "]" ] }, { @@ -75,8 +75,7 @@ "metadata": {}, "outputs": [], "source": [ - "from earthkit.data import ArrayField\n", - "from earthkit.data import SimpleFieldList\n", + "from earthkit.data import ArrayField, SimpleFieldList\n", "\n", "ds = SimpleFieldList()\n", "for f in d:\n", diff --git a/docs/examples/file_parts.ipynb b/docs/examples/file_parts.ipynb index 1da5b5600..19e5e9adc 100644 --- a/docs/examples/file_parts.ipynb +++ b/docs/examples/file_parts.ipynb @@ -793,10 +793,7 @@ } ], "source": [ - "ds = ekd.from_source(\"file\", [\n", - " [\"test.grib\", (0, 526)], \n", - " [\"test6.grib\", [(0, 240), (480, 240)]]\n", - " ])\n", + "ds = ekd.from_source(\"file\", [[\"test.grib\", (0, 526)], [\"test6.grib\", [(0, 240), (480, 240)]]])\n", "ds.ls()" ] }, @@ -924,10 +921,7 @@ } ], "source": [ - "ds = ekd.from_source(\"file\", [\n", - " [\"test.grib\", None], \n", - " [\"test6.grib\", [(0, 240), (480, 240)]]\n", - " ])\n", + "ds = ekd.from_source(\"file\", [[\"test.grib\", None], [\"test6.grib\", [(0, 240), (480, 240)]]])\n", "ds.ls()" ] }, diff --git a/docs/examples/file_target.ipynb b/docs/examples/file_target.ipynb index aaae65107..4b3f310e6 100644 --- a/docs/examples/file_target.ipynb +++ b/docs/examples/file_target.ipynb @@ -202,7 +202,7 @@ "out_file = \"_res_t1_file_handler.grib\"\n", "if os.path.isfile(out_file):\n", " os.remove(out_file)\n", - " \n", + "\n", "ds[0].to_target(\"file\", out_file, append=True)\n", "len(ekd.from_source(\"file\", out_file))" ] diff --git a/docs/examples/files.ipynb b/docs/examples/files.ipynb index e222bdb5a..68d16a369 100644 --- a/docs/examples/files.ipynb +++ b/docs/examples/files.ipynb @@ -57,8 +57,7 @@ }, "outputs": [], "source": [ - "ekd.download_example_file(\n", - " [\"test.grib\", \"test4.grib\", \"test6.grib\", \"test.nc\"])" + "ekd.download_example_file([\"test.grib\", \"test4.grib\", \"test6.grib\", \"test.nc\"])" ] }, { @@ -516,8 +515,7 @@ } ], "source": [ - "ds = ekd.from_source(\"file-pattern\", \"./test{id}.grib\",\n", - " {\"id\": [4, 6]})\n", + "ds = ekd.from_source(\"file-pattern\", \"./test{id}.grib\", {\"id\": [4, 6]})\n", "ds.ls()" ] }, diff --git a/docs/examples/from_object.ipynb b/docs/examples/from_object.ipynb index c59146763..302f71f65 100644 --- a/docs/examples/from_object.ipynb +++ b/docs/examples/from_object.ipynb @@ -31,7 +31,7 @@ "metadata": {}, "outputs": [], "source": [ - "demo_ndarray = np.random.rand(10, 10) * 10.\n", + "demo_ndarray = np.random.rand(10, 10) * 10.0\n", "demo_lats = np.arange(0, 91, 10)\n", "demo_lons = np.arange(0, 91, 10)\n", "# demo_ndarray" @@ -460,10 +460,12 @@ ], "source": [ "demo_da = xr.DataArray(\n", - " demo_ndarray, name=\"temperature\", dims={'latitude': demo_lats, 'longitude': demo_lons},\n", - " attrs={'units': 'Celsius', \"standard_name\": \"air_temperature\"}\n", + " demo_ndarray,\n", + " name=\"temperature\",\n", + " dims={\"latitude\": demo_lats, \"longitude\": demo_lons},\n", + " attrs={\"units\": \"Celsius\", \"standard_name\": \"air_temperature\"},\n", ")\n", - "demo_ds = demo_da.to_dataset().assign_attrs({'purpose': 'Demonstration', 'source': 'EarthkitExamples'})\n", + "demo_ds = demo_da.to_dataset().assign_attrs({\"purpose\": \"Demonstration\", \"source\": \"EarthkitExamples\"})\n", "\n", "demo_ds" ] diff --git a/docs/examples/grib_fdb_write.ipynb b/docs/examples/grib_fdb_write.ipynb index b7d4e808d..8e98e6850 100644 --- a/docs/examples/grib_fdb_write.ipynb +++ b/docs/examples/grib_fdb_write.ipynb @@ -92,11 +92,12 @@ "fdb_dir = \"./_fdb\"\n", "os.makedirs(fdb_dir, exist_ok=True)\n", "\n", - "config = {\"type\": \"local\",\n", - " \"engine\": \"toc\",\n", - " \"schema\": fdb_schema,\n", - " \"spaces\": [{\"handler\": \"Default\",\n", - " \"roots\": [{\"path\": fdb_dir}]}]}" + "config = {\n", + " \"type\": \"local\",\n", + " \"engine\": \"toc\",\n", + " \"schema\": fdb_schema,\n", + " \"spaces\": [{\"handler\": \"Default\", \"roots\": [{\"path\": fdb_dir}]}],\n", + "}" ] }, { @@ -329,17 +330,17 @@ ], "source": [ "request = {\n", - " 'class': 'od',\n", - " 'expver': '0001',\n", - " 'stream': 'oper',\n", - " 'date': '20180801',\n", - " 'time': 1200,\n", - " 'domain': 'g',\n", - " 'type': 'an',\n", - " 'levtype': 'pl',\n", - " 'levelist': 500,\n", - " 'step': 0,\n", - " 'param': [131, 132]\n", + " \"class\": \"od\",\n", + " \"expver\": \"0001\",\n", + " \"stream\": \"oper\",\n", + " \"date\": \"20180801\",\n", + " \"time\": 1200,\n", + " \"domain\": \"g\",\n", + " \"type\": \"an\",\n", + " \"levtype\": \"pl\",\n", + " \"levelist\": 500,\n", + " \"step\": 0,\n", + " \"param\": [131, 132],\n", "}\n", "\n", "# read fields from fdb as a stream\n", diff --git a/docs/examples/grib_indexing.ipynb b/docs/examples/grib_indexing.ipynb index 0b6cde31b..6b1418d0e 100644 --- a/docs/examples/grib_indexing.ipynb +++ b/docs/examples/grib_indexing.ipynb @@ -303,7 +303,7 @@ " a = fs.sel(gridType=\"regular_ll\")\n", " print(len(a))\n", "except KeyError as e:\n", - " print(f\"error: {e}\") " + " print(f\"error: {e}\")" ] }, { @@ -652,8 +652,7 @@ } ], "source": [ - "fs = ekd.from_source(\"file\", \"./_grib_dir_with_sql\", indexing=True, level=[500, 850], \n", - " order_by=\"variable\")\n", + "fs = ekd.from_source(\"file\", \"./_grib_dir_with_sql\", indexing=True, level=[500, 850], order_by=\"variable\")\n", "fs.ls()" ] } diff --git a/docs/examples/grib_metadata_object.ipynb b/docs/examples/grib_metadata_object.ipynb index 531ab6aa3..2928f4489 100644 --- a/docs/examples/grib_metadata_object.ipynb +++ b/docs/examples/grib_metadata_object.ipynb @@ -488,7 +488,7 @@ "v = ds.sel(param=\"v\", level=1000)[0]\n", "speed = np.sqrt(u.values**2 + v.values**2)\n", "md_speed = u.metadata().override(shortName=\"ws\")\n", - "ds_speed = FieldList.from_array(speed, md_speed) " + "ds_speed = FieldList.from_array(speed, md_speed)" ] }, { diff --git a/docs/examples/grib_nearest_gridpoint.ipynb b/docs/examples/grib_nearest_gridpoint.ipynb index faece631b..2f75eb8d3 100644 --- a/docs/examples/grib_nearest_gridpoint.ipynb +++ b/docs/examples/grib_nearest_gridpoint.ipynb @@ -29,10 +29,9 @@ "metadata": {}, "outputs": [], "source": [ - "import earthkit.data as ekd\n", - "from earthkit.geo import GeoKDTree\n", - "from earthkit.geo import nearest_point_haversine\n", - "from earthkit.geo import nearest_point_kdtree" + "from earthkit.geo import GeoKDTree, nearest_point_haversine, nearest_point_kdtree\n", + "\n", + "import earthkit.data as ekd" ] }, { @@ -158,10 +157,7 @@ } ], "source": [ - "p_ref = [\n", - " [51.45, 44.49, 50.73], \n", - " [-0.97, 18.34, -17.1]\n", - "]\n", + "p_ref = [[51.45, 44.49, 50.73], [-0.97, 18.34, -17.1]]\n", "idx, distance = nearest_point_kdtree(p_ref, (lat, lon))\n", "idx" ] @@ -405,10 +401,7 @@ } ], "source": [ - "p_ref = [\n", - " [51.45, 44.49, 50.73], \n", - " [-0.97, 18.34, -17.1]\n", - "]\n", + "p_ref = [[51.45, 44.49, 50.73], [-0.97, 18.34, -17.1]]\n", "idx, dist = nearest_point_haversine(p_ref, (lat, lon))\n", "idx" ] diff --git a/docs/examples/grib_time_series.ipynb b/docs/examples/grib_time_series.ipynb index 0688d29c2..4751c6440 100644 --- a/docs/examples/grib_time_series.ipynb +++ b/docs/examples/grib_time_series.ipynb @@ -42,8 +42,9 @@ }, "outputs": [], "source": [ - "import earthkit.data as ekd\n", - "from earthkit.geo import nearest_point_haversine" + "from earthkit.geo import nearest_point_haversine\n", + "\n", + "import earthkit.data as ekd" ] }, { diff --git a/docs/examples/grib_to_fdb_target.ipynb b/docs/examples/grib_to_fdb_target.ipynb index 5f727ef13..7dc0faf35 100644 --- a/docs/examples/grib_to_fdb_target.ipynb +++ b/docs/examples/grib_to_fdb_target.ipynb @@ -80,11 +80,12 @@ "fdb_dir = \"./_fdb_target_demo\"\n", "os.makedirs(fdb_dir, exist_ok=True)\n", "\n", - "config = {\"type\": \"local\",\n", - " \"engine\": \"toc\",\n", - " \"schema\": fdb_schema,\n", - " \"spaces\": [{\"handler\": \"Default\",\n", - " \"roots\": [{\"path\": fdb_dir}]}]}" + "config = {\n", + " \"type\": \"local\",\n", + " \"engine\": \"toc\",\n", + " \"schema\": fdb_schema,\n", + " \"spaces\": [{\"handler\": \"Default\", \"roots\": [{\"path\": fdb_dir}]}],\n", + "}" ] }, { @@ -383,17 +384,17 @@ "source": [ "# checking the result\n", "request = {\n", - " 'class': 'od',\n", - " 'expver': '0001',\n", - " 'stream': 'oper',\n", - " 'date': '20180801',\n", - " 'time': 1200,\n", - " 'domain': 'g',\n", - " 'type': 'an',\n", - " 'levtype': 'pl',\n", - " 'levelist': 500,\n", - " 'step': 0,\n", - " 'param': [131, 132]\n", + " \"class\": \"od\",\n", + " \"expver\": \"0001\",\n", + " \"stream\": \"oper\",\n", + " \"date\": \"20180801\",\n", + " \"time\": 1200,\n", + " \"domain\": \"g\",\n", + " \"type\": \"an\",\n", + " \"levtype\": \"pl\",\n", + " \"levelist\": 500,\n", + " \"step\": 0,\n", + " \"param\": [131, 132],\n", "}\n", "\n", "ekd.from_source(\"fdb\", request, config=config, stream=False).ls()" diff --git a/docs/examples/grib_to_zarr_target.ipynb b/docs/examples/grib_to_zarr_target.ipynb index b1c534652..3333d6ef1 100644 --- a/docs/examples/grib_to_zarr_target.ipynb +++ b/docs/examples/grib_to_zarr_target.ipynb @@ -210,11 +210,11 @@ ], "source": [ "# with these options each field will be a separate chunk\n", - "ds.to_target(\"zarr\", \n", - " earthkit_to_xarray_kwargs={\"chunks\": {\"forecast_reference_time\": 1, \n", - " \"step\": 1, \n", - " \"level\": 1}},\n", - " xarray_to_zarr_kwargs={\"store\": \"_pl.zarr\", \"mode\": \"w\"})" + "ds.to_target(\n", + " \"zarr\",\n", + " earthkit_to_xarray_kwargs={\"chunks\": {\"forecast_reference_time\": 1, \"step\": 1, \"level\": 1}},\n", + " xarray_to_zarr_kwargs={\"store\": \"_pl.zarr\", \"mode\": \"w\"},\n", + ")" ] }, { diff --git a/docs/examples/gribjump.ipynb b/docs/examples/gribjump.ipynb index 3cd2cf609..fa107a7a1 100644 --- a/docs/examples/gribjump.ipynb +++ b/docs/examples/gribjump.ipynb @@ -856,7 +856,7 @@ "source": [ "groups = source.sel(hdate=\"20200101\").group_by(\"time\")\n", "for group in groups:\n", - " print(group, group.to_numpy().shape, group.metadata('base_datetime'))" + " print(group, group.to_numpy().shape, group.metadata(\"base_datetime\"))" ] }, { diff --git a/docs/examples/list_of_dicts_geography.ipynb b/docs/examples/list_of_dicts_geography.ipynb index 78374beda..73025a53d 100644 --- a/docs/examples/list_of_dicts_geography.ipynb +++ b/docs/examples/list_of_dicts_geography.ipynb @@ -64,15 +64,15 @@ "\n", "def build_lod(data):\n", " prototype = {\n", - " **data,\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " \"param\": \"t\", \n", - " }\n", - " \n", + " **data,\n", + " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + " \"param\": \"t\",\n", + " }\n", + "\n", " lod = [\n", - " {\"level\": 500, **prototype},\n", - " {\"level\": 850, **prototype},\n", - " ]\n", + " {\"level\": 500, **prototype},\n", + " {\"level\": 850, **prototype},\n", + " ]\n", " return lod" ] }, @@ -180,9 +180,11 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0]),\n", - " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", + "data = {\n", + " \"latitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0]),\n", + " \"values\": np.array([1, 2, 3, 4, 5, 6]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds.ls()" @@ -271,9 +273,11 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0]),\n", - " \"values\": np.array([[1, 2], [3, 4], [5, 6]])}\n", + "data = {\n", + " \"latitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0]),\n", + " \"values\": np.array([[1, 2], [3, 4], [5, 6]]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" @@ -326,9 +330,11 @@ } ], "source": [ - "data = {\"distinctLatitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"distinctLongitudes\": np.array([20, 40.0]),\n", - " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", + "data = {\n", + " \"distinctLatitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"distinctLongitudes\": np.array([20, 40.0]),\n", + " \"values\": np.array([1, 2, 3, 4, 5, 6]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" @@ -404,9 +410,11 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10., 10., 0., 0., -10.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0, 20., 40., 20., 40.]),\n", - " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", + "data = {\n", + " \"latitudes\": np.array([10.0, 10.0, 0.0, 0.0, -10.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0, 20.0, 40.0, 20.0, 40.0]),\n", + " \"values\": np.array([1, 2, 3, 4, 5, 6]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" @@ -459,9 +467,11 @@ } ], "source": [ - "data = {\"latitudes\": np.array([[10., 10.], [0., 0.], [-10.0, -10.0]]),\n", - " \"longitudes\": np.array([[20.0, 40.], [20., 40.], [20., 40.]]),\n", - " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", + "data = {\n", + " \"latitudes\": np.array([[10.0, 10.0], [0.0, 0.0], [-10.0, -10.0]]),\n", + " \"longitudes\": np.array([[20.0, 40.0], [20.0, 40.0], [20.0, 40.0]]),\n", + " \"values\": np.array([1, 2, 3, 4, 5, 6]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" @@ -508,9 +518,11 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10., 10., 6., 6., 6.]),\n", - " \"longitudes\": np.array([20, 40.0, -40., 0., 40.]),\n", - " \"values\": np.array([1, 2, 3, 4, 5])}\n", + "data = {\n", + " \"latitudes\": np.array([10.0, 10.0, 6.0, 6.0, 6.0]),\n", + " \"longitudes\": np.array([20, 40.0, -40.0, 0.0, 40.0]),\n", + " \"values\": np.array([1, 2, 3, 4, 5]),\n", + "}\n", "\n", "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" diff --git a/docs/examples/list_of_dicts_overview.ipynb b/docs/examples/list_of_dicts_overview.ipynb index a9c312fbb..8b4a0c759 100644 --- a/docs/examples/list_of_dicts_overview.ipynb +++ b/docs/examples/list_of_dicts_overview.ipynb @@ -59,20 +59,20 @@ "import earthkit.data as ekd\n", "\n", "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0],\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + "}\n", "\n", "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 600, **prototype},\n", - " ]" + " {\"param\": \"t\", \"level\": 500, **prototype},\n", + " {\"param\": \"t\", \"level\": 850, **prototype},\n", + " {\"param\": \"u\", \"level\": 500, **prototype},\n", + " {\"param\": \"u\", \"level\": 850, **prototype},\n", + " {\"param\": \"d\", \"level\": 850, **prototype},\n", + " {\"param\": \"d\", \"level\": 600, **prototype},\n", + "]" ] }, { diff --git a/docs/examples/list_of_dicts_to_xarray.ipynb b/docs/examples/list_of_dicts_to_xarray.ipynb index 322f3097c..04f035880 100644 --- a/docs/examples/list_of_dicts_to_xarray.ipynb +++ b/docs/examples/list_of_dicts_to_xarray.ipynb @@ -1,968 +1,968 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "ee0f0104-8077-45f1-9746-58f29b64db92", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## list-of-dict: converting to Xarray" - ] - }, - { - "cell_type": "raw", - "id": "6cadbfbf-c7af-4927-8927-c320d9160c4f", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "This example demonstrates how :ref:`data-sources-lod` fieldlists can be converted into Xarray." - ] - }, - { - "cell_type": "markdown", - "id": "2e087423-8c96-49b4-984c-f15472fa8381", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Data containing geography" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "1e5ebf7a-2fc6-453a-9e14-6b04b5135810", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 248B\n",
-                            "Dimensions:    (levelist: 2, latitude: 3, longitude: 2)\n",
-                            "Coordinates:\n",
-                            "  * levelist   (levelist) int64 16B 500 850\n",
-                            "  * latitude   (latitude) float64 24B 10.0 0.0 -10.0\n",
-                            "  * longitude  (longitude) float64 16B 20.0 40.0\n",
-                            "Data variables:\n",
-                            "    t          (levelist, latitude, longitude) float64 96B ...\n",
-                            "    u          (levelist, latitude, longitude) float64 96B ...\n",
-                            "Attributes:\n",
-                            "    Conventions:  CF-1.8\n",
-                            "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 248B\n", - "Dimensions: (levelist: 2, latitude: 3, longitude: 2)\n", - "Coordinates:\n", - " * levelist (levelist) int64 16B 500 850\n", - " * latitude (latitude) float64 24B 10.0 0.0 -10.0\n", - " * longitude (longitude) float64 16B 20.0 40.0\n", - "Data variables:\n", - " t (levelist, latitude, longitude) float64 96B ...\n", - " u (levelist, latitude, longitude) float64 96B ...\n", - "Attributes:\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import earthkit.data as ekd\n", - "\n", - "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", - "\n", - "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " ]\n", - "\n", - "ds = ekd.from_source(\"list-of-dicts\", d)\n", - "ds.to_xarray()" - ] - }, - { - "cell_type": "markdown", - "id": "94b46ec8-614b-480a-8ffe-0b1dd4e344bb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Data without geography" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "7ea3d8bf-a432-4aef-94d9-5ac0c6b19503", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 208B\n",
-                            "Dimensions:   (levelist: 2, values: 6)\n",
-                            "Coordinates:\n",
-                            "  * levelist  (levelist) int64 16B 500 850\n",
-                            "Dimensions without coordinates: values\n",
-                            "Data variables:\n",
-                            "    t         (levelist, values) float64 96B ...\n",
-                            "    u         (levelist, values) float64 96B ...\n",
-                            "Attributes:\n",
-                            "    Conventions:  CF-1.8\n",
-                            "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 208B\n", - "Dimensions: (levelist: 2, values: 6)\n", - "Coordinates:\n", - " * levelist (levelist) int64 16B 500 850\n", - "Dimensions without coordinates: values\n", - "Data variables:\n", - " t (levelist, values) float64 96B ...\n", - " u (levelist, values) float64 96B ...\n", - "Attributes:\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "prototype = {\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", - "\n", - "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " ]\n", - "\n", - "ds = ekd.from_source(\"list-of-dicts\", d)\n", - "ds.to_xarray()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c6463409-7686-4d90-8cab-00a04b7119bb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } + "cells": [ + { + "cell_type": "markdown", + "id": "ee0f0104-8077-45f1-9746-58f29b64db92", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" }, - "nbformat": 4, - "nbformat_minor": 5 + "tags": [] + }, + "source": [ + "## list-of-dict: converting to Xarray" + ] + }, + { + "cell_type": "raw", + "id": "6cadbfbf-c7af-4927-8927-c320d9160c4f", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "This example demonstrates how :ref:`data-sources-lod` fieldlists can be converted into Xarray." + ] + }, + { + "cell_type": "markdown", + "id": "2e087423-8c96-49b4-984c-f15472fa8381", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Data containing geography" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1e5ebf7a-2fc6-453a-9e14-6b04b5135810", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 248B\n",
+       "Dimensions:    (levelist: 2, latitude: 3, longitude: 2)\n",
+       "Coordinates:\n",
+       "  * levelist   (levelist) int64 16B 500 850\n",
+       "  * latitude   (latitude) float64 24B 10.0 0.0 -10.0\n",
+       "  * longitude  (longitude) float64 16B 20.0 40.0\n",
+       "Data variables:\n",
+       "    t          (levelist, latitude, longitude) float64 96B ...\n",
+       "    u          (levelist, latitude, longitude) float64 96B ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.8\n",
+       "    institution:  ECMWF
" + ], + "text/plain": [ + " Size: 248B\n", + "Dimensions: (levelist: 2, latitude: 3, longitude: 2)\n", + "Coordinates:\n", + " * levelist (levelist) int64 16B 500 850\n", + " * latitude (latitude) float64 24B 10.0 0.0 -10.0\n", + " * longitude (longitude) float64 16B 20.0 40.0\n", + "Data variables:\n", + " t (levelist, latitude, longitude) float64 96B ...\n", + " u (levelist, latitude, longitude) float64 96B ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import earthkit.data as ekd\n", + "\n", + "prototype = {\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0],\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + "}\n", + "\n", + "d = [\n", + " {\"param\": \"t\", \"level\": 500, **prototype},\n", + " {\"param\": \"t\", \"level\": 850, **prototype},\n", + " {\"param\": \"u\", \"level\": 500, **prototype},\n", + " {\"param\": \"u\", \"level\": 850, **prototype},\n", + "]\n", + "\n", + "ds = ekd.from_source(\"list-of-dicts\", d)\n", + "ds.to_xarray()" + ] + }, + { + "cell_type": "markdown", + "id": "94b46ec8-614b-480a-8ffe-0b1dd4e344bb", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Data without geography" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7ea3d8bf-a432-4aef-94d9-5ac0c6b19503", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 208B\n",
+       "Dimensions:   (levelist: 2, values: 6)\n",
+       "Coordinates:\n",
+       "  * levelist  (levelist) int64 16B 500 850\n",
+       "Dimensions without coordinates: values\n",
+       "Data variables:\n",
+       "    t         (levelist, values) float64 96B ...\n",
+       "    u         (levelist, values) float64 96B ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.8\n",
+       "    institution:  ECMWF
" + ], + "text/plain": [ + " Size: 208B\n", + "Dimensions: (levelist: 2, values: 6)\n", + "Coordinates:\n", + " * levelist (levelist) int64 16B 500 850\n", + "Dimensions without coordinates: values\n", + "Data variables:\n", + " t (levelist, values) float64 96B ...\n", + " u (levelist, values) float64 96B ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prototype = {\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + "}\n", + "\n", + "d = [\n", + " {\"param\": \"t\", \"level\": 500, **prototype},\n", + " {\"param\": \"t\", \"level\": 850, **prototype},\n", + " {\"param\": \"u\", \"level\": 500, **prototype},\n", + " {\"param\": \"u\", \"level\": 850, **prototype},\n", + "]\n", + "\n", + "ds = ekd.from_source(\"list-of-dicts\", d)\n", + "ds.to_xarray()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6463409-7686-4d90-8cab-00a04b7119bb", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/docs/examples/mars.ipynb b/docs/examples/mars.ipynb index a4f9ca786..ebd47ecb7 100644 --- a/docs/examples/mars.ipynb +++ b/docs/examples/mars.ipynb @@ -34,11 +34,11 @@ "ds = ekd.from_source(\n", " \"mars\",\n", " request=dict(\n", - " param=[\"2t\", \"msl\"], \n", - " levtype=\"sfc\", \n", - " area=[50, -10, 40, 10], \n", - " grid=[2, 2], \n", - " date=\"2023-05-10\" # N,W,S,E\n", + " param=[\"2t\", \"msl\"],\n", + " levtype=\"sfc\",\n", + " area=[50, -10, 40, 10],\n", + " grid=[2, 2],\n", + " date=\"2023-05-10\", # N,W,S,E\n", " ),\n", ")" ] diff --git a/docs/examples/metadata.ipynb b/docs/examples/metadata.ipynb index ebc9dc83a..1bf4d2a67 100644 --- a/docs/examples/metadata.ipynb +++ b/docs/examples/metadata.ipynb @@ -101,7 +101,7 @@ "# from list of key/value pairs\n", "md = RawMetadata([(\"shortName\", \"2t\"), (\"perturbationNumber\", 5)])\n", "\n", - "# from keyword arguments \n", + "# from keyword arguments\n", "md = RawMetadata(shortName=\"2t\", perturbationNumber=5)" ] }, diff --git a/docs/examples/numpy_fieldlist.ipynb b/docs/examples/numpy_fieldlist.ipynb index 323f9cacf..107258de7 100644 --- a/docs/examples/numpy_fieldlist.ipynb +++ b/docs/examples/numpy_fieldlist.ipynb @@ -71,7 +71,7 @@ "def potential_temperature(t, p):\n", " # t: temperature in K\n", " # p: pressure in Pa\n", - " return t * (100000. / p)**0.285611" + " return t * (100000.0 / p) ** 0.285611" ] }, { @@ -149,7 +149,7 @@ "source": [ "t = f.values\n", "print(\"typeOfLevel=\", f.metadata(\"typeOfLevel\"))\n", - "p = f.metadata(\"level\") * 100. # hPa -> Pa\n", + "p = f.metadata(\"level\") * 100.0 # hPa -> Pa\n", "t_new = potential_temperature(t, p)\n", "t_new[:10]" ] @@ -556,7 +556,7 @@ } ], "source": [ - "p = np.asarray(fs.metadata(\"level\")).reshape(-1, 1) * 100. # hPa -> Pa\n", + "p = np.asarray(fs.metadata(\"level\")).reshape(-1, 1) * 100.0 # hPa -> Pa\n", "t_new = potential_temperature(fs.values, p)\n", "t_new.shape" ] @@ -860,10 +860,10 @@ "ds_r = FieldList()\n", "\n", "for f in fs:\n", - " p = f.metadata(\"level\") * 100. # hPa -> Pa\n", + " p = f.metadata(\"level\") * 100.0 # hPa -> Pa\n", " t_new = potential_temperature(f.values, p)\n", " md_new = f.metadata().override(shortName=\"pt\")\n", - " \n", + "\n", " # create new numpy fieldlist with a single field\n", " ds_new = FieldList.from_array(t_new, md_new)\n", "\n", diff --git a/docs/examples/pandas.ipynb b/docs/examples/pandas.ipynb index 8afa3243f..b617f2d02 100644 --- a/docs/examples/pandas.ipynb +++ b/docs/examples/pandas.ipynb @@ -231,10 +231,10 @@ } ], "source": [ - "t2m_series = pd.Series(np.linspace(273.15, 293, 20), name='t2m')\n", - "lat_series = pd.Series(np.arange(50, 52, 0.1), name='latitude')\n", - "lon_series = pd.Series(np.arange(-1, 1, 0.1), name='longitude')\n", - "date_series = pd.Series(pd.date_range('2022-01-01', '2022-01-20'), name='date')\n", + "t2m_series = pd.Series(np.linspace(273.15, 293, 20), name=\"t2m\")\n", + "lat_series = pd.Series(np.arange(50, 52, 0.1), name=\"latitude\")\n", + "lon_series = pd.Series(np.arange(-1, 1, 0.1), name=\"longitude\")\n", + "date_series = pd.Series(pd.date_range(\"2022-01-01\", \"2022-01-20\"), name=\"date\")\n", "\n", "date_series\n", "t2m_df = pd.concat([t2m_series, lat_series, lon_series], axis=1).set_index(date_series)\n", @@ -455,7 +455,7 @@ " print(thing)\n", "\n", "# Describe representation is a pandas dataframe\n", - "ek_pandas.describe()\n" + "ek_pandas.describe()" ] }, { diff --git a/docs/examples/polytope.ipynb b/docs/examples/polytope.ipynb index 84a5d4bed..3f8230564 100644 --- a/docs/examples/polytope.ipynb +++ b/docs/examples/polytope.ipynb @@ -141,17 +141,17 @@ "import earthkit.data as ekd\n", "\n", "request = {\n", - " 'stream': 'oper',\n", - " 'levtype': 'pl',\n", - " 'levellist': '500',\n", - " 'param': '129.128',\n", - " 'step': '0/12',\n", - " 'time': '00',\n", - " 'date': '20200915',\n", - " 'type': 'fc',\n", - " 'class': 'rd',\n", - " 'expver': 'hsvs',\n", - " 'domain': 'g'\n", + " \"stream\": \"oper\",\n", + " \"levtype\": \"pl\",\n", + " \"levellist\": \"500\",\n", + " \"param\": \"129.128\",\n", + " \"step\": \"0/12\",\n", + " \"time\": \"00\",\n", + " \"date\": \"20200915\",\n", + " \"type\": \"fc\",\n", + " \"class\": \"rd\",\n", + " \"expver\": \"hsvs\",\n", + " \"domain\": \"g\",\n", "}\n", "\n", "ds = ekd.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False)\n", diff --git a/docs/examples/polytope_feature.ipynb b/docs/examples/polytope_feature.ipynb index e95d09694..b74947e98 100644 --- a/docs/examples/polytope_feature.ipynb +++ b/docs/examples/polytope_feature.ipynb @@ -110,7 +110,7 @@ "source": [ "import earthkit.data\n", "\n", - "LOCATION = ((-9.11, 38.79))\n", + "LOCATION = (-9.11, 38.79)\n", "\n", "request = {\n", " \"class\": \"od\",\n", @@ -119,7 +119,7 @@ " \"date\": -1,\n", " \"time\": \"0000\",\n", " \"levtype\": \"sfc\",\n", - " \"expver\": \"0001\", \n", + " \"expver\": \"0001\",\n", " \"domain\": \"g\",\n", " \"param\": \"164/167/169\",\n", " \"number\": \"1/to/50\",\n", @@ -136,7 +136,7 @@ " \"ecmwf-mars\",\n", " request,\n", " stream=False,\n", - " address='polytope.ecmwf.int',\n", + " address=\"polytope.ecmwf.int\",\n", ")" ] }, @@ -787,7 +787,7 @@ "chart = Chart()\n", "chart.title(f\"ECMWF ensemble meteogram at {location_to_string(LOCATION)}\")\n", "chart.box(ds, time_frequency=TIME_FREQUENCY, quantiles=QUANTILES)\n", - "chart.line(ds, aggregation='mean', line_color='grey', time_frequency=TIME_FREQUENCY)\n", + "chart.line(ds, aggregation=\"mean\", line_color=\"grey\", time_frequency=TIME_FREQUENCY)\n", "chart.show(renderer=\"png\") # Replace with chart.show() in an interactive session!" ] }, @@ -1079,11 +1079,37 @@ " \"step\": \"0\",\n", " \"feature\": {\n", " \"type\": \"polygon\",\n", - " \"shape\": [[41.870881288, -8.8791360], [41.694339317422646, -8.824238614026456], [40.171924585721314, -8.902386975546364], [38.75694209400925, -9.493088042617785], [38.42424252381525, -9.171674240710018], [38.49907333213173, -8.676525850529856], [37.057269459205145, -8.971873318897366], [37.162874354643776, -7.406745406502978], [38.19776118392036, -6.931663452624974], [38.4280922170291, -7.321584397020473], [39.011852875635526, -6.9787177479519755], [39.66227871551288, -7.5393956904523804], [39.66568774825791, -7.03915852435145], [40.0019453234905, -6.883203763416162], [40.20373392742229, -7.035724907677206], [40.350463990828985, -6.8135246275213035], [41.030499770212515, -6.905947651233703], [41.593647729084154, -6.22847017956974], [41.67712153119277, -6.544984134823352], [41.949682257268876, -6.567927092516641], [41.96960294343674, -7.1747800681640115], [41.88337981339092, -7.196871678410446], [41.81334515396762, -8.156666519264604], [42.14242723772878, -8.205142297350534], [41.870881288, -8.8791360]],\n", + " \"shape\": [\n", + " [41.870881288, -8.8791360],\n", + " [41.694339317422646, -8.824238614026456],\n", + " [40.171924585721314, -8.902386975546364],\n", + " [38.75694209400925, -9.493088042617785],\n", + " [38.42424252381525, -9.171674240710018],\n", + " [38.49907333213173, -8.676525850529856],\n", + " [37.057269459205145, -8.971873318897366],\n", + " [37.162874354643776, -7.406745406502978],\n", + " [38.19776118392036, -6.931663452624974],\n", + " [38.4280922170291, -7.321584397020473],\n", + " [39.011852875635526, -6.9787177479519755],\n", + " [39.66227871551288, -7.5393956904523804],\n", + " [39.66568774825791, -7.03915852435145],\n", + " [40.0019453234905, -6.883203763416162],\n", + " [40.20373392742229, -7.035724907677206],\n", + " [40.350463990828985, -6.8135246275213035],\n", + " [41.030499770212515, -6.905947651233703],\n", + " [41.593647729084154, -6.22847017956974],\n", + " [41.67712153119277, -6.544984134823352],\n", + " [41.949682257268876, -6.567927092516641],\n", + " [41.96960294343674, -7.1747800681640115],\n", + " [41.88337981339092, -7.196871678410446],\n", + " [41.81334515396762, -8.156666519264604],\n", + " [42.14242723772878, -8.205142297350534],\n", + " [41.870881288, -8.8791360],\n", + " ],\n", " },\n", "}\n", "\n", - "ds = earthkit.data.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False, address='polytope.ecmwf.int')" + "ds = earthkit.data.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False, address=\"polytope.ecmwf.int\")" ] }, { @@ -1108,7 +1134,7 @@ "\n", "da = ds.to_xarray()\n", "chart = earthkit.plots.Map(domain=\"Portugal\")\n", - "chart.point_cloud(da['2t'], x=\"y\", y=\"x\")\n", + "chart.point_cloud(da[\"2t\"], x=\"y\", y=\"x\")\n", "\n", "chart.coastlines()\n", "chart.borders()\n", @@ -7224,15 +7250,20 @@ } ], "source": [ - "import earthkit\n", "import earthkit.geo.cartography\n", "import earthkit.plots\n", "\n", + "import earthkit\n", + "\n", "countries = [\"France\", \"Italy\", \"Spain\"] # List of countries\n", "\n", "shapes = earthkit.geo.cartography.country_polygons(countries, resolution=50e6)\n", "\n", - "request = {\"class\": \"od\", \"stream\": \"oper\", \"type\": \"fc\", \"levtype\": \"sfc\",\n", + "request = {\n", + " \"class\": \"od\",\n", + " \"stream\": \"oper\",\n", + " \"type\": \"fc\",\n", + " \"levtype\": \"sfc\",\n", " \"date\": 0,\n", " \"time\": 0,\n", " \"expver\": 1,\n", @@ -7244,7 +7275,7 @@ " },\n", "}\n", "\n", - "ds = earthkit.data.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False, address='polytope.ecmwf.int')" + "ds = earthkit.data.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False, address=\"polytope.ecmwf.int\")" ] }, { @@ -7268,7 +7299,7 @@ "xa = ds.to_xarray()\n", "\n", "chart = earthkit.plots.Map(domain=countries)\n", - "chart.point_cloud(xa['2t'], x=\"y\", y=\"x\")\n", + "chart.point_cloud(xa[\"2t\"], x=\"y\", y=\"x\")\n", "chart.coastlines()\n", "chart.borders()\n", "chart.gridlines()\n", diff --git a/docs/examples/polytope_time_series.ipynb b/docs/examples/polytope_time_series.ipynb index 0ea0722b6..1e7a51950 100644 --- a/docs/examples/polytope_time_series.ipynb +++ b/docs/examples/polytope_time_series.ipynb @@ -60,10 +60,11 @@ "\n", "location = [46.5, 17] # lat, lon\n", "\n", - "request = {\"class\": \"od\", \n", - " \"stream\": \"enfo\", \n", - " \"type\": \"pf\", \n", - " \"expver\": 1, \n", + "request = {\n", + " \"class\": \"od\",\n", + " \"stream\": \"enfo\",\n", + " \"type\": \"pf\",\n", + " \"expver\": 1,\n", " \"domain\": \"g\",\n", " \"date\": -1,\n", " \"time\": 0,\n", @@ -77,12 +78,12 @@ " \"range\": {\n", " \"start\": 0,\n", " \"end\": 360,\n", - " }\n", + " },\n", " },\n", " \"format\": \"covjson\",\n", "}\n", - " \n", - "ds = ekd.from_source(\"polytope\", \"ecmwf-mars\", request=request, stream=False, address='polytope.ecmwf.int')" + "\n", + "ds = ekd.from_source(\"polytope\", \"ecmwf-mars\", request=request, stream=False, address=\"polytope.ecmwf.int\")" ] }, { diff --git a/docs/examples/polytope_vertical_profile.ipynb b/docs/examples/polytope_vertical_profile.ipynb index 287a7ec29..74387bedc 100644 --- a/docs/examples/polytope_vertical_profile.ipynb +++ b/docs/examples/polytope_vertical_profile.ipynb @@ -80,7 +80,7 @@ " },\n", "}\n", "\n", - "ds = ekd.from_source(\"polytope\", \"ecmwf-mars\", request=request, stream=False, address='polytope.ecmwf.int')" + "ds = ekd.from_source(\"polytope\", \"ecmwf-mars\", request=request, stream=False, address=\"polytope.ecmwf.int\")" ] }, { diff --git a/docs/examples/s3.ipynb b/docs/examples/s3.ipynb index 2b7a514ae..487bbe9d1 100644 --- a/docs/examples/s3.ipynb +++ b/docs/examples/s3.ipynb @@ -242,12 +242,13 @@ "source": [ "import earthkit.data as ekd\n", "\n", - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\", \n", - " \"objects\": \"test6.grib\",\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": \"test6.grib\",\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, anon=True)\n", "ds.ls()" ] }, @@ -307,16 +308,17 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\", \n", - " \"objects\": \"test6.grib\",\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": \"test6.grib\",\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, stream=True, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, stream=True, anon=True)\n", "\n", "for f in ds:\n", " # f is GribField object. It gets deleted when going out of scope\n", - " print(f)\n" + " print(f)" ] }, { @@ -364,7 +366,7 @@ } ], "source": [ - "ds = ekd.from_source(\"s3\", req, stream=True, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, stream=True, anon=True)\n", "\n", "for f in ds.batched(2):\n", " # f is a fieldlist\n", @@ -540,7 +542,7 @@ } ], "source": [ - "ds = ekd.from_source(\"s3\", req, stream=True, read_all=True, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, stream=True, read_all=True, anon=True)\n", "ds.ls()" ] }, @@ -624,12 +626,13 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\", \n", - " \"objects\": [\"test6.grib\", \"tuv_pl.grib\"],\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": [\"test6.grib\", \"tuv_pl.grib\"],\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, anon=True)\n", "len(ds)" ] }, @@ -768,12 +771,13 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\",\n", - " \"objects\": {\"object\": \"test6.grib\", \"parts\": (240, 480)},\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": {\"object\": \"test6.grib\", \"parts\": (240, 480)},\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, anon=True)\n", "ds.ls()" ] }, @@ -883,12 +887,13 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\",\n", - " \"objects\": {\"object\": \"test6.grib\", \"parts\": [(0, 240), (480, 240)]},\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": {\"object\": \"test6.grib\", \"parts\": [(0, 240), (480, 240)]},\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, anon=True)\n", "ds.ls()" ] }, @@ -1040,13 +1045,13 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\", \n", - " \"objects\": [{\"object\": \"test6.grib\", \"parts\": (0, 240)}, \n", - " {\"object\": \"tuv_pl.grib\", \"parts\": (2400, 240)}],\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": [{\"object\": \"test6.grib\", \"parts\": (0, 240)}, {\"object\": \"tuv_pl.grib\", \"parts\": (2400, 240)}],\n", + "}\n", "\n", - "ds = ekd.from_source(\"s3\", req, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, anon=True)\n", "ds.ls()" ] }, @@ -1101,13 +1106,14 @@ } ], "source": [ - "req = {\"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", - " \"bucket\": \"earthkit-test-data-public\",\n", - " \"objects\": {\"object\": \"test6.grib\", \"parts\": (240, 480)},\n", - " }\n", + "req = {\n", + " \"endpoint\": \"object-store.os-api.cci1.ecmwf.int\",\n", + " \"bucket\": \"earthkit-test-data-public\",\n", + " \"objects\": {\"object\": \"test6.grib\", \"parts\": (240, 480)},\n", + "}\n", "\n", "\n", - "ds = ekd.from_source(\"s3\", req, stream=True, anon=True) \n", + "ds = ekd.from_source(\"s3\", req, stream=True, anon=True)\n", "\n", "for f in ds:\n", " # f is GribField object. It gets deleted when going out of scope\n", diff --git a/docs/examples/ukmo_pp.ipynb b/docs/examples/ukmo_pp.ipynb index 86d099be4..10f084d43 100644 --- a/docs/examples/ukmo_pp.ipynb +++ b/docs/examples/ukmo_pp.ipynb @@ -65,7 +65,7 @@ "source": [ "import earthkit.data as ekd\n", "\n", - "# we ensure the example file is available. \n", + "# we ensure the example file is available.\n", "ekd.download_example_file(\"air_temp.pp\")\n", "\n", "ds = ekd.from_source(\"file\", \"air_temp.pp\")" diff --git a/docs/examples/url.ipynb b/docs/examples/url.ipynb index 97d1760b0..68d0cbbdc 100644 --- a/docs/examples/url.ipynb +++ b/docs/examples/url.ipynb @@ -67,8 +67,7 @@ "source": [ "import earthkit.data as ekd\n", "\n", - "fs = ekd.from_source(\"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\")" + "fs = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\")" ] }, { @@ -210,8 +209,7 @@ } ], "source": [ - "fs = ekd.from_source(\"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test_gribs.tar\")" + "fs = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test_gribs.tar\")" ] }, { @@ -539,9 +537,13 @@ } ], "source": [ - "fs = ekd.from_source(\"url\", \n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\",\n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\"])\n", + "fs = ekd.from_source(\n", + " \"url\",\n", + " [\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\",\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\",\n", + " ],\n", + ")\n", "fs.ls()" ] }, @@ -822,9 +824,9 @@ } ], "source": [ - "fs = ekd.from_source(\"url-pattern\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test{id}.grib\",\n", - " {\"id\": [4, 6]})\n", + "fs = ekd.from_source(\n", + " \"url-pattern\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test{id}.grib\", {\"id\": [4, 6]}\n", + ")\n", "fs.ls()" ] }, @@ -967,10 +969,11 @@ "import datetime\n", "\n", "fs = ekd.from_source(\n", - " \"url-pattern\", \n", + " \"url-pattern\",\n", " \"https://sites.ecmwf.int/repository/earthkit-data/test-data/test_{my_date:date(%Y-%m-%d)}_{name}.grib\",\n", - " {\"my_date\": datetime.datetime(2020, 5, 13), \"name\": [\"t2\", \"msl\"]})\n", - "fs.ls()\n" + " {\"my_date\": datetime.datetime(2020, 5, 13), \"name\": [\"t2\", \"msl\"]},\n", + ")\n", + "fs.ls()" ] }, { diff --git a/docs/examples/url_parts.ipynb b/docs/examples/url_parts.ipynb index 106eaf68b..83420560f 100644 --- a/docs/examples/url_parts.ipynb +++ b/docs/examples/url_parts.ipynb @@ -233,9 +233,7 @@ } ], "source": [ - "ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\")\n", + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\")\n", "ds.ls(extra_keys=\"offset\")" ] }, @@ -349,10 +347,7 @@ } ], "source": [ - "ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " parts=(0, 240))\n", + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", parts=(0, 240))\n", "ds.ls()" ] }, @@ -449,10 +444,7 @@ } ], "source": [ - "ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " parts=[(0, 240)])\n", + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", parts=[(0, 240)])\n", "ds.ls()" ] }, @@ -549,10 +541,7 @@ } ], "source": [ - "ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " parts=[(0, 245)])\n", + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", parts=[(0, 245)])\n", "ds.ls()" ] }, @@ -694,9 +683,8 @@ ], "source": [ "ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " parts=[(0, 240), (480, 480)])\n", + " \"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", parts=[(0, 240), (480, 480)]\n", + ")\n", "ds.ls()" ] }, @@ -731,9 +719,8 @@ "source": [ "try:\n", " ds = ekd.from_source(\n", - " \"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " parts=[(0, 240), (220, 240)])\n", + " \"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", parts=[(0, 240), (220, 240)]\n", + " )\n", "except Exception as e:\n", " print(e)" ] @@ -876,11 +863,13 @@ } ], "source": [ - "ds = ekd.from_source(\"url\", [\n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\", (0, 526)], \n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", \n", - " [(0, 240), (480, 240)]]\n", - " ])\n", + "ds = ekd.from_source(\n", + " \"url\",\n", + " [\n", + " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\", (0, 526)],\n", + " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", [(0, 240), (480, 240)]],\n", + " ],\n", + ")\n", "ds.ls()" ] }, @@ -1023,11 +1012,12 @@ ], "source": [ "ds = ekd.from_source(\n", - " \"url\", [\n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\", None], \n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", \n", - " [(0, 240), (480, 240)]]\n", - " ])\n", + " \"url\",\n", + " [\n", + " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test.grib\", None],\n", + " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\", [(0, 240), (480, 240)]],\n", + " ],\n", + ")\n", "ds.ls()" ] }, @@ -1140,10 +1130,13 @@ ], "source": [ "ds = ekd.from_source(\n", - " \"url\", \n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/tuv_pl.grib\"], \n", - " parts=(0, 240))\n", + " \"url\",\n", + " [\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/tuv_pl.grib\",\n", + " ],\n", + " parts=(0, 240),\n", + ")\n", "ds.ls()" ] }, diff --git a/docs/examples/url_stream.ipynb b/docs/examples/url_stream.ipynb index 04e9a566d..eacb9e34c 100644 --- a/docs/examples/url_stream.ipynb +++ b/docs/examples/url_stream.ipynb @@ -59,9 +59,7 @@ }, "outputs": [], "source": [ - "ds = ekd.from_source(\"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", \n", - " stream=True)" + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", stream=True)" ] }, { @@ -144,9 +142,7 @@ } ], "source": [ - "ds = ekd.from_source(\"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", \n", - " stream=True)\n", + "ds = ekd.from_source(\"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", stream=True)\n", "\n", "for f in ds.batched(2):\n", " # f is a fieldlist\n", @@ -194,9 +190,9 @@ } ], "source": [ - "ds = ekd.from_source(\"url\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", \n", - " stream=True, read_all=True)\n", + "ds = ekd.from_source(\n", + " \"url\", \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", stream=True, read_all=True\n", + ")\n", "\n", "len(ds)" ] @@ -372,10 +368,14 @@ } ], "source": [ - "ds = ekd.from_source(\"url\", \n", - " [\"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\", \n", - " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\"], \n", - " stream=True)\n", + "ds = ekd.from_source(\n", + " \"url\",\n", + " [\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test4.grib\",\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/examples/test6.grib\",\n", + " ],\n", + " stream=True,\n", + ")\n", "\n", "for f in ds.batched(3):\n", " # f is a fieldlist\n", diff --git a/docs/examples/wekeo.ipynb b/docs/examples/wekeo.ipynb index d9ad225eb..75cb508aa 100644 --- a/docs/examples/wekeo.ipynb +++ b/docs/examples/wekeo.ipynb @@ -745,14 +745,14 @@ "import earthkit.data as ekd\n", "\n", "ds = ekd.from_source(\n", - " \"wekeo\",\n", - " \"EO:CLMS:DAT:CLMS_GLOBAL_BA_300M_V3_MONTHLY_NETCDF\",\n", - " request={\n", - " \"dataset_id\": \"EO:CLMS:DAT:CLMS_GLOBAL_BA_300M_V3_MONTHLY_NETCDF\",\n", - " \"startdate\": \"2019-01-01T00:00:00.000Z\",\n", - " \"enddate\": \"2019-01-01T23:59:59.999Z\",\n", - " }\n", - " )\n", + " \"wekeo\",\n", + " \"EO:CLMS:DAT:CLMS_GLOBAL_BA_300M_V3_MONTHLY_NETCDF\",\n", + " request={\n", + " \"dataset_id\": \"EO:CLMS:DAT:CLMS_GLOBAL_BA_300M_V3_MONTHLY_NETCDF\",\n", + " \"startdate\": \"2019-01-01T00:00:00.000Z\",\n", + " \"enddate\": \"2019-01-01T23:59:59.999Z\",\n", + " },\n", + ")\n", "\n", "ds.to_xarray()" ] diff --git a/docs/examples/xarray_cupy.ipynb b/docs/examples/xarray_cupy.ipynb index 9d5d1f897..c666ac325 100644 --- a/docs/examples/xarray_cupy.ipynb +++ b/docs/examples/xarray_cupy.ipynb @@ -609,9 +609,9 @@ "metadata": {}, "outputs": [], "source": [ - "r_cp = r.earthkit.to_device(\"cuda:0\") \n", + "r_cp = r.earthkit.to_device(\"cuda:0\")\n", "# equivalent code:\n", - "# r_cp = r.earthkit.to_device(\"cuda:0\", array_backend=\"cupy\") " + "# r_cp = r.earthkit.to_device(\"cuda:0\", array_backend=\"cupy\")" ] }, { @@ -1153,7 +1153,7 @@ "source": [ "r_np = r_cp.earthkit.to_device(\"cpu\")\n", "# equivalent code:\n", - "# r_np = r.earthkit.to_device(\"cpu\", array_backend=\"numpy\") " + "# r_np = r.earthkit.to_device(\"cpu\", array_backend=\"numpy\")" ] }, { diff --git a/docs/examples/xarray_engine_chunks.ipynb b/docs/examples/xarray_engine_chunks.ipynb index 22ff23e9e..0a83c96bc 100644 --- a/docs/examples/xarray_engine_chunks.ipynb +++ b/docs/examples/xarray_engine_chunks.ipynb @@ -724,9 +724,7 @@ } ], "source": [ - "ds = ds_fl.to_xarray(time_dim_mode=\"valid_time\", \n", - " chunks={\"valid_time\": 10}, \n", - " add_earthkit_attrs=False)\n", + "ds = ds_fl.to_xarray(time_dim_mode=\"valid_time\", chunks={\"valid_time\": 10}, add_earthkit_attrs=False)\n", "ds[\"2t\"]" ] }, diff --git a/docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb b/docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb index 36235c737..d1f1e92c3 100644 --- a/docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb +++ b/docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb @@ -1122,9 +1122,7 @@ } ], "source": [ - "ds = ds_fl.to_xarray(time_dim_mode=\"valid_time\", \n", - " chunks={\"valid_time\": 10}, \n", - " add_earthkit_attrs=False)\n", + "ds = ds_fl.to_xarray(time_dim_mode=\"valid_time\", chunks={\"valid_time\": 10}, add_earthkit_attrs=False)\n", "ds[\"2t\"]" ] }, diff --git a/docs/examples/xarray_engine_dims_as_attrs.ipynb b/docs/examples/xarray_engine_dims_as_attrs.ipynb index 1bbe1ea3b..b62b57557 100644 --- a/docs/examples/xarray_engine_dims_as_attrs.ipynb +++ b/docs/examples/xarray_engine_dims_as_attrs.ipynb @@ -738,9 +738,9 @@ ], "source": [ "ds = ds_fl.to_xarray(\n", - " profile=None, \n", - " dims_as_attrs=[\"level\", \"level_type\"], \n", - " add_earthkit_attrs=False, \n", + " profile=None,\n", + " dims_as_attrs=[\"level\", \"level_type\"],\n", + " add_earthkit_attrs=False,\n", ")\n", "ds" ] @@ -1306,10 +1306,10 @@ ], "source": [ "ds2 = ds_fl.to_xarray(\n", - " profile=None, \n", - " level_dim_mode=\"level_per_type\", \n", - " dims_as_attrs=\"\", \n", - " add_earthkit_attrs=False, \n", + " profile=None,\n", + " level_dim_mode=\"level_per_type\",\n", + " dims_as_attrs=\"\",\n", + " add_earthkit_attrs=False,\n", ")\n", "ds2" ] @@ -2087,10 +2087,10 @@ ], "source": [ "ds3 = ds_fl2.to_xarray(\n", - " profile=None, \n", - " level_dim_mode=\"level_per_type\", \n", - " dims_as_attrs=[\"\"], \n", - " add_earthkit_attrs=False, \n", + " profile=None,\n", + " level_dim_mode=\"level_per_type\",\n", + " dims_as_attrs=[\"\"],\n", + " add_earthkit_attrs=False,\n", ")\n", "ds3" ] @@ -2662,9 +2662,9 @@ ], "source": [ "ds4 = ds_fl2.to_xarray(\n", - " profile=None, \n", - " dims_as_attrs=[\"level\", \"level_type\"], \n", - " add_earthkit_attrs=False, \n", + " profile=None,\n", + " dims_as_attrs=[\"level\", \"level_type\"],\n", + " add_earthkit_attrs=False,\n", ")\n", "ds4" ] diff --git a/docs/examples/xarray_engine_extra_dims.ipynb b/docs/examples/xarray_engine_extra_dims.ipynb index a3a590b95..fc327a62b 100644 --- a/docs/examples/xarray_engine_extra_dims.ipynb +++ b/docs/examples/xarray_engine_extra_dims.ipynb @@ -869,9 +869,9 @@ ], "source": [ "ds = ds_fl.to_xarray(\n", - " profile=\"grib\", \n", - " extra_dims=[\"directionNumber\", \"frequencyNumber\"], \n", - " add_earthkit_attrs=False, \n", + " profile=\"grib\",\n", + " extra_dims=[\"directionNumber\", \"frequencyNumber\"],\n", + " add_earthkit_attrs=False,\n", ")\n", "ds" ] @@ -1399,12 +1399,9 @@ ], "source": [ "ds2 = ds_fl.to_xarray(\n", - " profile=\"grib\", \n", - " extra_dims=[\n", - " {\"d\": \"directionNumber\"}, \n", - " {\"f\": \"frequencyNumber\"}\n", - " ],\n", - " add_earthkit_attrs=False, \n", + " profile=\"grib\",\n", + " extra_dims=[{\"d\": \"directionNumber\"}, {\"f\": \"frequencyNumber\"}],\n", + " add_earthkit_attrs=False,\n", ")\n", "ds2" ] @@ -1770,16 +1767,18 @@ } ], "source": [ - "ds_fl2.ls(keys=[\n", - " \"shortName\", \n", - " \"dataDate\", \n", - " \"dataTime\", \n", - " \"stepRange\", \n", - " \"dataType\", \n", - " \"quantile\", \n", - " \"number\", \n", - " \"numberOfForecastsInEnsemble\"\n", - "])" + "ds_fl2.ls(\n", + " keys=[\n", + " \"shortName\",\n", + " \"dataDate\",\n", + " \"dataTime\",\n", + " \"stepRange\",\n", + " \"dataType\",\n", + " \"quantile\",\n", + " \"number\",\n", + " \"numberOfForecastsInEnsemble\",\n", + " ]\n", + ")" ] }, { @@ -2311,11 +2310,11 @@ ], "source": [ "ds3 = ds_fl2.to_xarray(\n", - " profile=\"grib\", \n", - " squeeze=False, \n", - " extra_dims=\"quantile\", \n", - " drop_dims=\"number\", \n", - " add_earthkit_attrs=False, \n", + " profile=\"grib\",\n", + " squeeze=False,\n", + " extra_dims=\"quantile\",\n", + " drop_dims=\"number\",\n", + " add_earthkit_attrs=False,\n", ")\n", "ds3" ] @@ -2841,11 +2840,11 @@ ], "source": [ "ds4 = ds_fl2.sel(quantile=\"2:3\").to_xarray(\n", - " profile=\"grib\", \n", - " squeeze=True, \n", - " ensure_dims=\"quantile\", \n", - " drop_dims=\"number\", \n", - " add_earthkit_attrs=False, \n", + " profile=\"grib\",\n", + " squeeze=True,\n", + " ensure_dims=\"quantile\",\n", + " drop_dims=\"number\",\n", + " add_earthkit_attrs=False,\n", ")\n", "ds4" ] diff --git a/docs/examples/xarray_engine_holes.ipynb b/docs/examples/xarray_engine_holes.ipynb index 83417641f..bf09a22ab 100644 --- a/docs/examples/xarray_engine_holes.ipynb +++ b/docs/examples/xarray_engine_holes.ipynb @@ -116,6 +116,7 @@ " ds_fl.to_xarray()\n", "except ValueError as e:\n", " import textwrap\n", + "\n", " print(textwrap.fill(str(e), 100))" ] }, diff --git a/docs/examples/xarray_engine_mono_variable.ipynb b/docs/examples/xarray_engine_mono_variable.ipynb index 90da7a6c5..fd8d8590f 100644 --- a/docs/examples/xarray_engine_mono_variable.ipynb +++ b/docs/examples/xarray_engine_mono_variable.ipynb @@ -827,12 +827,13 @@ } ], "source": [ - "ds = ds_fl.to_xarray(fixed_dims=[\"valid_time\", \"param\"],\n", - " mono_variable=True,\n", - " chunks={\"valid_time\": 1}, \n", - " flatten_values=True, \n", - " add_earthkit_attrs=False, \n", - " )\n", + "ds = ds_fl.to_xarray(\n", + " fixed_dims=[\"valid_time\", \"param\"],\n", + " mono_variable=True,\n", + " chunks={\"valid_time\": 1},\n", + " flatten_values=True,\n", + " add_earthkit_attrs=False,\n", + ")\n", "ds" ] }, @@ -2350,13 +2351,14 @@ } ], "source": [ - "ds = ds_fl.to_xarray(fixed_dims=[\"valid_time\", \"param\", \"number\"],\n", - " mono_variable=True,\n", - " chunks={\"valid_time\": 1}, \n", - " flatten_values=True, \n", - " add_earthkit_attrs=False, \n", - " fill_metadata={\"number\": 0},\n", - " )\n", + "ds = ds_fl.to_xarray(\n", + " fixed_dims=[\"valid_time\", \"param\", \"number\"],\n", + " mono_variable=True,\n", + " chunks={\"valid_time\": 1},\n", + " flatten_values=True,\n", + " add_earthkit_attrs=False,\n", + " fill_metadata={\"number\": 0},\n", + ")\n", "ds" ] } diff --git a/docs/examples/xarray_engine_mono_variable_remapping.ipynb b/docs/examples/xarray_engine_mono_variable_remapping.ipynb index 8c59446dd..4b66af8e4 100644 --- a/docs/examples/xarray_engine_mono_variable_remapping.ipynb +++ b/docs/examples/xarray_engine_mono_variable_remapping.ipynb @@ -795,13 +795,14 @@ } ], "source": [ - "ds = ds_fl.to_xarray(fixed_dims=[\"valid_time\", \"param\", \"number\"],\n", - " mono_variable=True,\n", - " chunks={\"valid_time\": 1}, \n", - " flatten_values=True, \n", - " add_earthkit_attrs=False, \n", - " remapping={\"param\": \"{param}_{level}\"}\n", - " )\n", + "ds = ds_fl.to_xarray(\n", + " fixed_dims=[\"valid_time\", \"param\", \"number\"],\n", + " mono_variable=True,\n", + " chunks={\"valid_time\": 1},\n", + " flatten_values=True,\n", + " add_earthkit_attrs=False,\n", + " remapping={\"param\": \"{param}_{level}\"},\n", + ")\n", "ds" ] }, diff --git a/docs/examples/xarray_engine_overview.ipynb b/docs/examples/xarray_engine_overview.ipynb index b5773de15..9d8f5962c 100644 --- a/docs/examples/xarray_engine_overview.ipynb +++ b/docs/examples/xarray_engine_overview.ipynb @@ -838,7 +838,7 @@ } ], "source": [ - "m_0 = ds_fl.sel(param=\"t\", step=6, level=500)[0].values.mean() \n", + "m_0 = ds_fl.sel(param=\"t\", step=6, level=500)[0].values.mean()\n", "m_1 = ds_fl1.sel(param=\"t\", step=6, level=500)[0].values.mean()\n", "m_0, m_1" ] diff --git a/docs/examples/xarray_engine_remapping.ipynb b/docs/examples/xarray_engine_remapping.ipynb index 171332fb5..1c3c2c29c 100644 --- a/docs/examples/xarray_engine_remapping.ipynb +++ b/docs/examples/xarray_engine_remapping.ipynb @@ -721,10 +721,7 @@ ], "source": [ "ds = ds_fl.to_xarray(\n", - " profile=None,\n", - " remapping={\"member\": \"{dataType}_{number}\"}, \n", - " extra_dims=\"member\", \n", - " add_earthkit_attrs=False\n", + " profile=None, remapping={\"member\": \"{dataType}_{number}\"}, extra_dims=\"member\", add_earthkit_attrs=False\n", ")\n", "ds" ] @@ -1241,14 +1238,14 @@ "source": [ "ds2 = ds_fl.to_xarray(\n", " profile=None,\n", - " squeeze=True, \n", + " squeeze=True,\n", " remapping={\n", - " \"member\": \"{dataType}_{number}\", \n", - " \"mars\": \"{class}_{stream}\", \n", - " }, \n", - " extra_dims=[\"member\", \"mars\"], \n", - " dims_as_attrs=\"mars\", \n", - " add_earthkit_attrs=False\n", + " \"member\": \"{dataType}_{number}\",\n", + " \"mars\": \"{class}_{stream}\",\n", + " },\n", + " extra_dims=[\"member\", \"mars\"],\n", + " dims_as_attrs=\"mars\",\n", + " add_earthkit_attrs=False,\n", ")\n", "ds2" ] @@ -2074,10 +2071,10 @@ ], "source": [ "ds3 = ds_fl2.to_xarray(\n", - " profile=\"grib\", \n", - " remapping={\"my_custom_var_key\": \"{param}__{typeOfLevel}_{level}\"}, \n", + " profile=\"grib\",\n", + " remapping={\"my_custom_var_key\": \"{param}__{typeOfLevel}_{level}\"},\n", " variable_key=\"my_custom_var_key\",\n", - " add_earthkit_attrs=False\n", + " add_earthkit_attrs=False,\n", ")\n", "ds3" ] @@ -2603,11 +2600,11 @@ ], "source": [ "ds4 = ds_fl2.to_xarray(\n", - " profile=\"grib\", \n", - " level_dim_mode=\"level_per_type\", \n", - " remapping={\"my_custom_var_key\": \"{param}_{typeOfLevel}\"}, \n", - " variable_key=\"my_custom_var_key\", \n", - " add_earthkit_attrs=False\n", + " profile=\"grib\",\n", + " level_dim_mode=\"level_per_type\",\n", + " remapping={\"my_custom_var_key\": \"{param}_{typeOfLevel}\"},\n", + " variable_key=\"my_custom_var_key\",\n", + " add_earthkit_attrs=False,\n", ")\n", "ds4" ] diff --git a/docs/examples/xarray_engine_seasonal.ipynb b/docs/examples/xarray_engine_seasonal.ipynb index 9a6216894..0e0fccf48 100644 --- a/docs/examples/xarray_engine_seasonal.ipynb +++ b/docs/examples/xarray_engine_seasonal.ipynb @@ -666,8 +666,7 @@ } ], "source": [ - "ds = ds_fl.to_xarray(time_dim_mode=\"forecast\", \n", - " dim_roles={\"step\": \"forecastMonth\"})\n", + "ds = ds_fl.to_xarray(time_dim_mode=\"forecast\", dim_roles={\"step\": \"forecastMonth\"})\n", "ds" ] }, @@ -1173,9 +1172,7 @@ } ], "source": [ - "ds = ds_fl.to_xarray(time_dim_mode=\"forecast\", \n", - " dim_roles={\"step\": \"forecastMonth\"}, \n", - " dim_name_from_role_name=False)\n", + "ds = ds_fl.to_xarray(time_dim_mode=\"forecast\", dim_roles={\"step\": \"forecastMonth\"}, dim_name_from_role_name=False)\n", "ds" ] } diff --git a/docs/examples/xarray_engine_step_ranges.ipynb b/docs/examples/xarray_engine_step_ranges.ipynb index bfdfcbc8f..1dba30f2d 100644 --- a/docs/examples/xarray_engine_step_ranges.ipynb +++ b/docs/examples/xarray_engine_step_ranges.ipynb @@ -594,7 +594,7 @@ ], "source": [ "# convert to hours from ns\n", - "[int(x * 1E-9 / (3600)) for x in ds[\"step\"].values]" + "[int(x * 1e-9 / (3600)) for x in ds[\"step\"].values]" ] }, { @@ -636,7 +636,7 @@ ], "source": [ "ds = ds_fl.to_xarray(dim_roles={\"step\": \"startStep\"})\n", - "[int(x * 1E-9 / (3600)) for x in ds[\"step\"].values]" + "[int(x * 1e-9 / (3600)) for x in ds[\"step\"].values]" ] }, { diff --git a/docs/experimental/ekd_test_on_cds_era5_single_level.ipynb b/docs/experimental/ekd_test_on_cds_era5_single_level.ipynb index 9496abf2b..54c5f1d24 100644 --- a/docs/experimental/ekd_test_on_cds_era5_single_level.ipynb +++ b/docs/experimental/ekd_test_on_cds_era5_single_level.ipynb @@ -50,7 +50,10 @@ } ], "source": [ - "fl = ekd.from_source('url', 'https://sites.ecmwf.int/repository/earthkit-data/test-data/xr_engine/cds-reanalysis-era5-single-levels-20230101-low-resol.grib')" + "fl = ekd.from_source(\n", + " \"url\",\n", + " \"https://sites.ecmwf.int/repository/earthkit-data/test-data/xr_engine/cds-reanalysis-era5-single-levels-20230101-low-resol.grib\",\n", + ")" ] }, { @@ -84,7 +87,7 @@ } ], "source": [ - "fl.unique_values('edition', 'stream', 'dataType', 'stepType', 'edition', 'gridType', 'Ni')" + "fl.unique_values(\"edition\", \"stream\", \"dataType\", \"stepType\", \"edition\", \"gridType\", \"Ni\")" ] }, { @@ -469,8 +472,22 @@ } ], "source": [ - "fl_ls = fl.ls(extra_keys=['stream', 'stepType', 'step', 'Ni', 'Nj', 'validityDate', 'validityTime', \n", - " 'gridType', 'md5GridSection', 'bitmapPresent', 'gridSpec', 'edition'])\n", + "fl_ls = fl.ls(\n", + " extra_keys=[\n", + " \"stream\",\n", + " \"stepType\",\n", + " \"step\",\n", + " \"Ni\",\n", + " \"Nj\",\n", + " \"validityDate\",\n", + " \"validityTime\",\n", + " \"gridType\",\n", + " \"md5GridSection\",\n", + " \"bitmapPresent\",\n", + " \"gridSpec\",\n", + " \"edition\",\n", + " ]\n", + ")\n", "fl_ls" ] }, @@ -562,7 +579,7 @@ } ], "source": [ - "fl_ls[['edition', 'Ni', 'md5GridSection']].value_counts().reset_index().sort_values('Ni')" + "fl_ls[[\"edition\", \"Ni\", \"md5GridSection\"]].value_counts().reset_index().sort_values(\"Ni\")" ] }, { @@ -773,7 +790,9 @@ } ], "source": [ - "fl_ls[['dataDate', 'dataTime', 'stepType', 'step', 'stepRange', 'validityDate', 'validityTime']].value_counts().reset_index()" + "fl_ls[\n", + " [\"dataDate\", \"dataTime\", \"stepType\", \"step\", \"stepRange\", \"validityDate\", \"validityTime\"]\n", + "].value_counts().reset_index()" ] }, { @@ -796,7 +815,7 @@ } ], "source": [ - "fl_ls['number'].value_counts()" + "fl_ls[\"number\"].value_counts()" ] }, { @@ -826,9 +845,9 @@ ], "source": [ "dss, split_coords_list = fl.to_xarray(\n", - " split_dims=['stream', 'dataType', 'edition', 'Ni'], \n", - " time_dim_mode='valid_time', \n", - " squeeze=False, \n", + " split_dims=[\"stream\", \"dataType\", \"edition\", \"Ni\"],\n", + " time_dim_mode=\"valid_time\",\n", + " squeeze=False,\n", ")\n", "len(dss)" ] diff --git a/docs/experimental/target_fdb_grib.ipynb b/docs/experimental/target_fdb_grib.ipynb index adda8a88e..e0720b3e0 100644 --- a/docs/experimental/target_fdb_grib.ipynb +++ b/docs/experimental/target_fdb_grib.ipynb @@ -45,11 +45,12 @@ "fdb_dir = \"./_fdb\"\n", "os.makedirs(fdb_dir, exist_ok=True)\n", "\n", - "config = {\"type\": \"local\",\n", - " \"engine\": \"toc\",\n", - " \"schema\": fdb_schema,\n", - " \"spaces\": [{\"handler\": \"Default\",\n", - " \"roots\": [{\"path\": fdb_dir}]}]}\n", + "config = {\n", + " \"type\": \"local\",\n", + " \"engine\": \"toc\",\n", + " \"schema\": fdb_schema,\n", + " \"spaces\": [{\"handler\": \"Default\", \"roots\": [{\"path\": fdb_dir}]}],\n", + "}\n", "\n", "# os.environ[\"FDB5_CONFIG\"] = str(c)\n", "\n", @@ -113,22 +114,22 @@ "import pyfdb\n", "\n", "request = {\n", - " 'class': 'od',\n", - " 'expver': '0001',\n", - " 'stream': 'oper',\n", - " 'date': '20200513',\n", - " 'time': 1200,\n", - " 'domain': 'g',\n", - " 'type': 'an',\n", - " 'levtype': 'sfc',\n", - " 'step': 0,\n", - " 'param': [167, 151]\n", + " \"class\": \"od\",\n", + " \"expver\": \"0001\",\n", + " \"stream\": \"oper\",\n", + " \"date\": \"20200513\",\n", + " \"time\": 1200,\n", + " \"domain\": \"g\",\n", + " \"type\": \"an\",\n", + " \"levtype\": \"sfc\",\n", + " \"step\": 0,\n", + " \"param\": [167, 151],\n", "}\n", "\n", "fdb = pyfdb.FDB(config=config)\n", "\n", "for el in fdb.list(request, True, True):\n", - " print(el['keys'])" + " print(el[\"keys\"])" ] }, { diff --git a/docs/experimental/target_file_grib.ipynb b/docs/experimental/target_file_grib.ipynb index 425ce18e9..6228ff6fa 100644 --- a/docs/experimental/target_file_grib.ipynb +++ b/docs/experimental/target_file_grib.ipynb @@ -320,8 +320,7 @@ "from earthkit.data.encoders.grib import GribEncoder\n", "from earthkit.data.targets.file import FileTarget\n", "\n", - "target = FileTarget(\"res2.grib\", \n", - " encoder=GribEncoder(metadata={\"type\": \"fc\", \"step\": 6}))\n", + "target = FileTarget(\"res2.grib\", encoder=GribEncoder(metadata={\"type\": \"fc\", \"step\": 6}))\n", "target.write(ds)\n", "target.close()\n", "\n", diff --git a/docs/experimental/target_file_png.ipynb b/docs/experimental/target_file_png.ipynb index 63f0147cf..27c4a885d 100644 --- a/docs/experimental/target_file_png.ipynb +++ b/docs/experimental/target_file_png.ipynb @@ -21,7 +21,8 @@ "# helper\n", "def show_img(fname):\n", " from IPython.display import Image\n", - " return Image(filename=fname, width=300) " + "\n", + " return Image(filename=fname, width=300)" ] }, { @@ -96,6 +97,7 @@ ], "source": [ "from earthkit.data.encoders.png import PngEncoder\n", + "\n", "from earthkit.data.targets.file import FileTarget\n", "\n", "target = FileTarget(\"res1.png\", encoder=PngEncoder())\n", diff --git a/environment.yml b/environment.yml index 9ae4ceb1e..8b66a0113 100644 --- a/environment.yml +++ b/environment.yml @@ -1,63 +1,63 @@ name: earthkit-data channels: - - conda-forge - - nodefaults +- conda-forge +- nodefaults dependencies: - - eccodes=>2.35.0 - - python-eccodes>=1.7.0 - - pip - - numpy - - pandas - - xarray>=0.19.0 - - dask - - netcdf4 - - cfgrib>=0.9.10.1 - - pdbufr>=0.11.0 - - pyodc - - filelock - - pyyaml - - entrypoints - - jupyterlab - - ecmwf-api-client>=1.6.1 - - cdsapi>=0.7.2 - - hda>=2.22 - - pip: - - multiurl>=0.3.3 - - pyfdb>=0.1.0 - - ecmwf-opendata>=0.1.2 - - polytope-client>=0.7.4 - - covjsonkit>=0.2.2 - - earthkit-geo>=0.2.0 - - earthkit-utils>=0.2.0 - - tqdm>=4.63.0 - - lru-dict - - markdown - - aws-requests-auth - - botocore - - make - - mypy - - myst-parser - - pre-commit - - pydata-sphinx-theme - - pyfakefs - - pytest - - pytest-cov - - pytest-forked - - pytest-reraise - - pytest-timeout - - sphinx>=7.2.6 - - pip: - - sphinx-autoapi>=3.0.0 - - sphinx_rtd_theme - - sphinxcontrib-apidoc - - sphinx-issues - - sphinx-tabs - - sphinx-copybutton - - nbformat - - nbconvert - - nbsphinx - - ipykernel - - geopandas - - deprecation - - ncdata - - scitools-iris +- eccodes=>2.35.0 +- python-eccodes>=1.7.0 +- pip +- numpy +- pandas +- xarray>=0.19.0 +- dask +- netcdf4 +- cfgrib>=0.9.10.1 +- pdbufr>=0.11.0 +- pyodc +- filelock +- pyyaml +- entrypoints +- jupyterlab +- ecmwf-api-client>=1.6.1 +- cdsapi>=0.7.2 +- hda>=2.22 +- pip: + - multiurl>=0.3.3 + - pyfdb>=0.1.0 + - ecmwf-opendata>=0.1.2 + - polytope-client>=0.7.4 + - covjsonkit>=0.2.2 + - earthkit-geo>=0.2.0 + - earthkit-utils>=0.2.0 +- tqdm>=4.63.0 +- lru-dict +- markdown +- aws-requests-auth +- botocore +- make +- mypy +- myst-parser +- pre-commit +- pydata-sphinx-theme +- pyfakefs +- pytest +- pytest-cov +- pytest-forked +- pytest-reraise +- pytest-timeout +- sphinx>=7.2.6 +- pip: + - sphinx-autoapi>=3.0.0 +- sphinx_rtd_theme +- sphinxcontrib-apidoc +- sphinx-issues +- sphinx-tabs +- sphinx-copybutton +- nbformat +- nbconvert +- nbsphinx +- ipykernel +- geopandas +- deprecation +- ncdata +- scitools-iris diff --git a/pyproject.toml b/pyproject.toml index 9a93dd1f6..e0624f52b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,10 @@ [build-system] -requires = [ "setuptools>=61", "setuptools-scm>=8" ] +requires = ["setuptools>=61", "setuptools-scm>=8"] [project] -name = "earthkit-data" -description = "A format-agnostic Python interface for geospatial data" -readme = "README.md" -license = { text = "Apache License Version 2.0" } authors = [ - { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, + {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int"} ] -requires-python = ">=3.10" classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", @@ -21,9 +16,8 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: Python :: Implementation :: PyPy" ] -dynamic = [ "version" ] dependencies = [ "cfgrib>=0.9.10.1", "dask", @@ -42,16 +36,23 @@ dependencies = [ "pdbufr>=0.11", "pyyaml", "tqdm>=4.63", - "xarray>=0.19", + "xarray>=0.19" ] +description = "A format-agnostic Python interface for geospatial data" +dynamic = ["version"] +license = {text = "Apache License Version 2.0"} +name = "earthkit-data" +readme = "README.md" +requires-python = ">=3.10" +entry-points."xarray.backends".earthkit = "earthkit.data.utils.xarray.engine:EarthkitBackendEntrypoint" optional-dependencies.all = [ - "earthkit-data[cds,covjsonkit,ecmwf-opendata,fdb,geo,geopandas,gribjump,iris,mars,odb,polytope,projection,s3,wekeo]", + "earthkit-data[cds,covjsonkit,ecmwf-opendata,fdb,geo,geopandas,gribjump,iris,mars,odb,polytope,projection,s3,wekeo]" ] -optional-dependencies.cds = [ "cdsapi>=0.7.2" ] -optional-dependencies.ci = [ "numpy" ] -optional-dependencies.covjsonkit = [ "covjsonkit>=0.2.2" ] +optional-dependencies.cds = ["cdsapi>=0.7.2"] +optional-dependencies.ci = ["numpy"] +optional-dependencies.covjsonkit = ["covjsonkit>=0.2.2"] optional-dependencies.dev = [ - "earthkit-data[all,docs,test,geotiff,iris]", + "earthkit-data[all,docs,test,geotiff,iris]" ] optional-dependencies.docs = [ "nbsphinx", @@ -61,20 +62,20 @@ optional-dependencies.docs = [ "sphinx-copybutton", "sphinx-issues", "sphinx-rtd-theme", - "sphinx-tabs", + "sphinx-tabs" ] -optional-dependencies.ecmwf-opendata = [ "ecmwf-opendata>=0.3.3" ] -optional-dependencies.fdb = [ "pyfdb>=0.1" ] -optional-dependencies.geo = [ "earthkit-geo>=0.2" ] -optional-dependencies.geopandas = [ "geopandas" ] -optional-dependencies.geotiff = [ "pyproj", "rasterio", "rioxarray" ] -optional-dependencies.gribjump = [ "pyfdb>=0.1", "pygribjump" ] -optional-dependencies.iris = [ "ncdata>=0.3.2", "scitools-iris" ] -optional-dependencies.mars = [ "ecmwf-api-client>=1.6.1" ] -optional-dependencies.odb = [ "pyodc" ] -optional-dependencies.polytope = [ "polytope-client>=0.7.6" ] -optional-dependencies.projection = [ "cartopy" ] -optional-dependencies.s3 = [ "aws-requests-auth", "botocore" ] +optional-dependencies.ecmwf-opendata = ["ecmwf-opendata>=0.3.3"] +optional-dependencies.fdb = ["pyfdb>=0.1"] +optional-dependencies.geo = ["earthkit-geo>=0.2"] +optional-dependencies.geopandas = ["geopandas"] +optional-dependencies.geotiff = ["pyproj", "rasterio", "rioxarray"] +optional-dependencies.gribjump = ["pyfdb>=0.1", "pygribjump"] +optional-dependencies.iris = ["ncdata>=0.3.2", "scitools-iris"] +optional-dependencies.mars = ["ecmwf-api-client>=1.6.1"] +optional-dependencies.odb = ["pyodc"] +optional-dependencies.polytope = ["polytope-client>=0.7.6"] +optional-dependencies.projection = ["cartopy"] +optional-dependencies.s3 = ["aws-requests-auth", "botocore"] optional-dependencies.test = [ "earthkit-data-demo-source", "nbconvert", @@ -84,33 +85,52 @@ optional-dependencies.test = [ "pytest-cov", "pytest-forked", "pytest-reraise", - "pytest-timeout", + "pytest-timeout" ] -optional-dependencies.wekeo = [ "hda>=2.22" ] -optional-dependencies.zarr = [ "zarr>=3" ] +optional-dependencies.wekeo = ["hda>=2.22"] +optional-dependencies.zarr = ["zarr>=3"] urls.Documentation = "https://earthkit-data.readthedocs.io/" urls.Homepage = "https://github.com/ecmwf/earthkit-data/" urls.Issues = "https://github.com/ecmwf/earthkit-data.issues" urls.Repository = "https://github.com/ecmwf/earthkit-data/" -entry-points."xarray.backends".earthkit = "earthkit.data.utils.xarray.engine:EarthkitBackendEntrypoint" - -[tool.setuptools.packages.find] -include = [ "earthkit.data" ] -where = [ "src/" ] +[tool.coverage.run] +branch = "true" -[tool.setuptools_scm] -version_file = "src/earthkit/data/_version.py" +[tool.pydocstyle] +add_ignore = ["D1", "D200", "D205", "D400", "D401"] +convention = "numpy" [tool.ruff] line-length = 120 +preview = true -lint.select = [ "E", "F", "I" ] -lint.isort.force-single-line = true +[tool.ruff.lint] +ignore = [ + "D1", # pydocstyle: Missing Docstrings + "D107", # pydocstyle: numpy convention + "D203", + "D205", + "D212", + "D213", + "D401", + "D402", + "D413", + "D415", + "D416", + "D417" +] +select = [ + "F", # pyflakes + "E", # pycodestyle + "W", # pycodestyle warnings + "I", # isort + "D" # pydocstyle +] -[tool.coverage.run] -branch = "true" +[tool.setuptools.packages.find] +include = ["earthkit.data"] +where = ["src/"] -[tool.pydocstyle] -add_ignore = [ "D1", "D200", "D205", "D400", "D401" ] -convention = "numpy" +[tool.setuptools_scm] +version_file = "src/earthkit/data/_version.py" diff --git a/src/earthkit/data/__init__.py b/src/earthkit/data/__init__.py index 87f693d03..3d09a0640 100644 --- a/src/earthkit/data/__init__.py +++ b/src/earthkit/data/__init__.py @@ -23,19 +23,14 @@ from .arguments.transformers import ALL from .core.caching import CACHE as cache from .core.config import CONFIG as config -from .core.fieldlist import Field -from .core.fieldlist import FieldList +from .core.fieldlist import Field, FieldList from .encoders import create_encoder from .indexing.fieldlist import SimpleFieldList from .readers.grib.output import new_grib_output -from .sources import Source -from .sources import from_source -from .sources import from_source_lazily +from .sources import Source, from_source, from_source_lazily from .sources.array_list import ArrayField -from .targets import create_target -from .targets import to_target -from .utils.examples import download_example_file -from .utils.examples import remote_example_file +from .targets import create_target, to_target +from .utils.examples import download_example_file, remote_example_file settings = config diff --git a/src/earthkit/data/arguments/args_kwargs.py b/src/earthkit/data/arguments/args_kwargs.py index b49a44ac2..478304b4c 100644 --- a/src/earthkit/data/arguments/args_kwargs.py +++ b/src/earthkit/data/arguments/args_kwargs.py @@ -73,7 +73,7 @@ def add_default_values_and_kwargs(self): self.kwargs = new_kwargs def ensure_positionals_only(self): - """Move positional arguments from self.kwargs into self.args""" + """Move positional arguments from self.kwargs into self.args.""" for name in self.positionals_only: value = self.kwargs.pop(name) self.args.append(value) diff --git a/src/earthkit/data/arguments/argument.py b/src/earthkit/data/arguments/argument.py index 46ab57672..1c6d82702 100644 --- a/src/earthkit/data/arguments/argument.py +++ b/src/earthkit/data/arguments/argument.py @@ -9,9 +9,7 @@ import logging from earthkit.data.arguments.earthkit_types import infer_type -from earthkit.data.arguments.transformers import AliasTransformer -from earthkit.data.arguments.transformers import FormatTransformer -from earthkit.data.arguments.transformers import TypeTransformer +from earthkit.data.arguments.transformers import AliasTransformer, FormatTransformer, TypeTransformer LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/arguments/input_manager.py b/src/earthkit/data/arguments/input_manager.py index 874185297..3922e1970 100644 --- a/src/earthkit/data/arguments/input_manager.py +++ b/src/earthkit/data/arguments/input_manager.py @@ -9,8 +9,7 @@ import logging from .argument import Argument -from .transformers import AvailabilityChecker -from .transformers import KwargsAliasTransformer +from .transformers import AvailabilityChecker, KwargsAliasTransformer LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/arguments/transformers.py b/src/earthkit/data/arguments/transformers.py index 17dbd4998..b9d915b80 100644 --- a/src/earthkit/data/arguments/transformers.py +++ b/src/earthkit/data/arguments/transformers.py @@ -169,9 +169,9 @@ def execute_before_default(self, kwargs): new_kwargs = {} for k, v in kwargs.items(): new_k = self.reversed_aliases.get(k, k) - assert ( - new_k not in new_kwargs - ), f"Error: Multiple values were given for aliased arguments: with '{k}' and '{new_k}'." + assert new_k not in new_kwargs, ( + f"Error: Multiple values were given for aliased arguments: with '{k}' and '{new_k}'." + ) new_kwargs[new_k] = v return new_kwargs @@ -181,7 +181,7 @@ def reversed_aliases(self): for target, aliases in self.aliases.items(): for alias in aliases: assert alias not in reversed, ( - "Error: Multiple target value for alias " f" argument '{alias}': '{target}' and '{reversed[alias]}'" + f"Error: Multiple target value for alias argument '{alias}': '{target}' and '{reversed[alias]}'" ) reversed[alias] = target return reversed diff --git a/src/earthkit/data/conf/gridspec.yaml b/src/earthkit/data/conf/gridspec.yaml index 879c13bfa..aacc1b709 100644 --- a/src/earthkit/data/conf/gridspec.yaml +++ b/src/earthkit/data/conf/gridspec.yaml @@ -44,69 +44,69 @@ grib_key_map: definitions: _1: &area area: - - north - - west - - south - - east + - north + - west + - south + - east _2: &rotation rotation: - - lat_south_pole - - lon_south_pole + - lat_south_pole + - lon_south_pole _3: first_point: - - first_lat - - first_lon + - first_lat + - first_lon shared_keys: &shared_keys - - type - - *rotation - - angle_of_rotation - - j_points_consecutive - - i_scans_negatively - - j_scans_positively +- type +- *rotation +- angle_of_rotation +- j_points_consecutive +- i_scans_negatively +- j_scans_positively rotation_keys: - - rotation - - angle_of_rotation +- rotation +- angle_of_rotation types: sh: grid_type: sh spec: - - type - - J - - K - - M + - type + - J + - K + - M regular_ll: grid_type: regular_ll rotated_type: rotated_ll spec: - - *shared_keys - - grid: - - dx - - dy - - *area + - *shared_keys + - grid: + - dx + - dy + - *area reduced_ll: grid_type: reduced_ll spec: - - *shared_keys - - grid: - - dy - - *area + - *shared_keys + - grid: + - dy + - *area regular_gg: grid_type: regular_gg rotated_type: rotated_gg spec: - - *shared_keys - - grid: - - N - - *area + - *shared_keys + - grid: + - N + - *area N_label: F reduced_gg: grid_type: reduced_gg rotated_type: reduced_rotated_gg spec: - - *shared_keys - - grid: - - N - - *area + - *shared_keys + - grid: + - N + - *area N_label: octahedral: 0: N @@ -164,7 +164,7 @@ types: healpix: grid_type: healpix spec: - - type - - grid: - - H - - order + - type + - grid: + - H + - order diff --git a/src/earthkit/data/core/__init__.py b/src/earthkit/data/core/__init__.py index 4329f4330..68a87c961 100644 --- a/src/earthkit/data/core/__init__.py +++ b/src/earthkit/data/core/__init__.py @@ -36,17 +36,17 @@ def patch(cls, obj, *args, **kwargs): class Base(metaclass=MetaBase): # Convertors def to_numpy(self, **kwargs): - """Convert into a numpy array""" + """Convert into a numpy array.""" self._not_implemented() @abstractmethod def to_xarray(self, **kwargs): - """Convert into an xarray dataset""" + """Convert into an xarray dataset.""" self._not_implemented() @abstractmethod def to_pandas(self, **kwargs): - """Convert into a pandas dataframe""" + """Convert into a pandas dataframe.""" self._not_implemented() # Change class @@ -95,7 +95,7 @@ def order_by(self, *args, **kwargs): def unique_values(self, *coords, remapping=None, patches=None, progress_bar=False): """Given a list of metadata attributes, such as date, param, levels, - returns the list of unique values for each attributes + returns the list of unique values for each attributes. """ from earthkit.data.core.order import build_remapping @@ -170,9 +170,9 @@ def _not_implemented(self): raise NotImplementedError(f"{module}.{name}.{func}(){extra}") def batched(self, *args): - """Return iterator for batches of data""" + """Return iterator for batches of data.""" self._not_implemented() def group_by(self, *args): - """Return iterator for batches of data grouped by metadata keys""" + """Return iterator for batches of data grouped by metadata keys.""" self._not_implemented() diff --git a/src/earthkit/data/core/caching.py b/src/earthkit/data/core/caching.py index 9ba845718..2de027487 100644 --- a/src/earthkit/data/core/caching.py +++ b/src/earthkit/data/core/caching.py @@ -24,8 +24,7 @@ import sqlite3 import threading import time -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from copy import deepcopy from random import randrange @@ -73,8 +72,7 @@ def __init__(self, path): def __repr__(self): return ( - f"DiskUsage(total={self.total},free={self.free}," - f"avail={self.avail},percent={self.percent},path={self.path})" + f"DiskUsage(total={self.total},free={self.free},avail={self.avail},percent={self.percent},path={self.path})" ) @@ -451,7 +449,7 @@ def _decache(self, bytes, purge=False): LOG.warning("earthkit-data cache: could not free %s", humanize.bytes(bytes)) def _register_cache_file(self, path, owner, args, parent=None): - """Register a file in the cache + """Register a file in the cache. Parameters ---------- diff --git a/src/earthkit/data/core/config.py b/src/earthkit/data/core/config.py index 8656bac42..d82dbe5e2 100644 --- a/src/earthkit/data/core/config.py +++ b/src/earthkit/data/core/config.py @@ -13,8 +13,7 @@ import os import tempfile import warnings -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from contextlib import contextmanager from typing import Callable @@ -23,11 +22,7 @@ from earthkit.data import __version__ as VERSION from earthkit.data.utils.html import css -from earthkit.data.utils.humanize import as_bytes -from earthkit.data.utils.humanize import as_percent -from earthkit.data.utils.humanize import as_seconds -from earthkit.data.utils.humanize import interval_to_human -from earthkit.data.utils.humanize import list_to_human +from earthkit.data.utils.humanize import as_bytes, as_percent, as_seconds, interval_to_human, list_to_human from earthkit.data.utils.interval import Interval LOG = logging.getLogger(__name__) @@ -267,7 +262,7 @@ def validate(self, name, value): @contextmanager def new_config(s): - """Context manager to create new config""" + """Context manager to create new config.""" CONFIG._stack.append(s) CONFIG._notify() try: @@ -334,7 +329,7 @@ def __init__(self, config_yaml: str, defaults: dict, callbacks=[]): @forward def get(self, name: str, default=NONE): - """[summary] + """[summary]. Parameters ---------- @@ -372,7 +367,7 @@ def get(self, name: str, default=NONE): @forward def set(self, *args, **kwargs): - """[summary] + """[summary]. Parameters ---------- @@ -395,7 +390,7 @@ def set(self, *args, **kwargs): self._changed() def _set(self, name: str, *args, **kwargs): - """[summary] + """[summary]. Parameters ---------- diff --git a/src/earthkit/data/core/fieldlist.py b/src/earthkit/data/core/fieldlist.py index 4d166bbbf..e7d489ec9 100644 --- a/src/earthkit/data/core/fieldlist.py +++ b/src/earthkit/data/core/fieldlist.py @@ -13,19 +13,15 @@ from collections import defaultdict import deprecation +from earthkit.utils.array import array_namespace as eku_array_namespace +from earthkit.utils.array import convert as convert_array +from earthkit.utils.array.convert import convert_dtype from earthkit.data.core import Base -from earthkit.data.core.index import Index -from earthkit.data.core.index import MaskIndex -from earthkit.data.core.index import MultiIndex +from earthkit.data.core.index import Index, MaskIndex, MultiIndex from earthkit.data.core.order import build_remapping -from earthkit.data.decorators import cached_method -from earthkit.data.decorators import detect_out_filename -from earthkit.data.decorators import thread_safe_cached_property +from earthkit.data.decorators import cached_method, detect_out_filename, thread_safe_cached_property from earthkit.data.utils.metadata.args import metadata_argument -from earthkit.utils.array import array_namespace as eku_array_namespace -from earthkit.utils.array import convert as convert_array -from earthkit.utils.array.convert import convert_dtype def _bits_per_value_to_metadata(**kwargs): @@ -239,6 +235,13 @@ def data(self, keys=("lat", "lon", "value"), flatten=False, dtype=None, index=No of the field is used. When ``keys`` is a single value only the array belonging to the key is returned. + See Also + -------- + to_latlon + to_points + to_numpy + values + Examples -------- - :ref:`/examples/grib_lat_lon_value.ipynb` @@ -260,13 +263,6 @@ def data(self, keys=("lat", "lon", "value"), flatten=False, dtype=None, index=No >>> d[0, 0] # first longitude 0.0 - See Also - -------- - to_latlon - to_points - to_numpy - values - """ _keys = dict( lat=self._metadata.geography.latitudes, @@ -634,7 +630,6 @@ def metadata(self, *keys, astype=None, remapping=None, patches=None, **kwargs): >>> r["name"] '2 metre temperature' """ - if remapping is not None or patches is not None: remapping = build_remapping(remapping, patches) return remapping(self.metadata)(*keys, astype=astype, **kwargs) @@ -767,7 +762,7 @@ def default_encoder(self): return self._metadata.data_format() def _encode(self, encoder, **kwargs): - """Double dispatch to the encoder""" + """Double dispatch to the encoder.""" return encoder._encode_field(self, **kwargs) def __getitem__(self, key): @@ -952,7 +947,7 @@ def _to_fieldlist(self): @staticmethod def _flatten(v): - """Flatten the array without copying the data." + """Flatten the array without copying the data. Parameters ---------- @@ -997,7 +992,7 @@ def __init__(self, **kwargs): import warnings warnings.warn( - ("array_backend option is not supported any longer in FieldList!" " Use to_fieldlist() instead"), + ("array_backend option is not supported any longer in FieldList! Use to_fieldlist() instead"), DeprecationWarning, ) kwargs.pop("array_backend", None) @@ -1274,6 +1269,13 @@ def data( ValueError When not all the fields have the same grid geometry. + See Also + -------- + to_latlon + to_points + to_numpy + values + Examples -------- - :ref:`/examples/grib_lat_lon_value.ipynb` @@ -1298,13 +1300,6 @@ def data( >>> d[0, 0, 0] # first longitude 0.0 - See Also - -------- - to_latlon - to_points - to_numpy - values - """ if isinstance(keys, str): keys = [keys] @@ -1750,7 +1745,7 @@ def default_encoder(self): return self[0]._metadata.data_format() def _encode(self, encoder, **kwargs): - """Double dispatch to the encoder""" + """Double dispatch to the encoder.""" return encoder._encode_fieldlist(self, **kwargs) def to_tensor(self, *args, **kwargs): @@ -1801,12 +1796,9 @@ def to_fieldlist(self, array_backend=None, array_namespace=None, device=None, ** dtype('float32') """ - return self.from_fields( - [ - f.copy(array_backend=array_backend, array_namespace=array_namespace, device=device, **kwargs) - for f in self - ] - ) + return self.from_fields([ + f.copy(array_backend=array_backend, array_namespace=array_namespace, device=device, **kwargs) for f in self + ]) def cube(self, *args, **kwargs): from earthkit.data.indexing.cube import FieldCube @@ -1823,7 +1815,7 @@ def merge(cls, sources): return MultiFieldList(sources) def _cache_diag(self): - """For testing only""" + """For testing only.""" from earthkit.data.utils.diag import metadata_cache_diag return metadata_cache_diag(self) diff --git a/src/earthkit/data/core/geography.py b/src/earthkit/data/core/geography.py index 373c437e4..a91e9e361 100644 --- a/src/earthkit/data/core/geography.py +++ b/src/earthkit/data/core/geography.py @@ -8,12 +8,11 @@ # -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod class Geography(metaclass=ABCMeta): - """Geographical information about a field or data unit""" + """Geographical information about a field or data unit.""" @abstractmethod def latitudes(self, dtype=None): diff --git a/src/earthkit/data/core/gridspec.py b/src/earthkit/data/core/gridspec.py index ca4a1477b..fa0f7a215 100644 --- a/src/earthkit/data/core/gridspec.py +++ b/src/earthkit/data/core/gridspec.py @@ -11,6 +11,6 @@ class GridSpec(RawMetadata): - """Geographical information about a field or data unit""" + """Geographical information about a field or data unit.""" pass diff --git a/src/earthkit/data/core/index.py b/src/earthkit/data/core/index.py index e9cb3eb4f..f4a9ad98d 100644 --- a/src/earthkit/data/core/index.py +++ b/src/earthkit/data/core/index.py @@ -14,10 +14,8 @@ from collections import defaultdict import earthkit.data -from earthkit.data.core.order import build_remapping -from earthkit.data.core.order import normalize_order_by -from earthkit.data.core.select import normalize_selection -from earthkit.data.core.select import selection_from_index +from earthkit.data.core.order import build_remapping, normalize_order_by +from earthkit.data.core.select import normalize_selection, selection_from_index from earthkit.data.sources import Source LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/core/ipython.py b/src/earthkit/data/core/ipython.py index 90f5fa2c1..43089b696 100644 --- a/src/earthkit/data/core/ipython.py +++ b/src/earthkit/data/core/ipython.py @@ -7,7 +7,7 @@ # nor does it submit to any jurisdiction. # -"""ipython is not None when running a notebook""" +"""ipython is not None when running a notebook.""" import logging import sys @@ -106,11 +106,8 @@ def _identity(x, **kwargs): if ipython_active: - from IPython.display import HTML + from IPython.display import HTML, Image, Markdown, display from IPython.display import SVG as SVG_ipython - from IPython.display import Image - from IPython.display import Markdown - from IPython.display import display def SVG(*args, **kwargs): import IPython diff --git a/src/earthkit/data/core/metadata.py b/src/earthkit/data/core/metadata.py index 6583afa63..21342fbe0 100644 --- a/src/earthkit/data/core/metadata.py +++ b/src/earthkit/data/core/metadata.py @@ -8,8 +8,7 @@ # import functools -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod class MetadataAccessor: diff --git a/src/earthkit/data/core/plugins.py b/src/earthkit/data/core/plugins.py index ec381b597..1e4185d93 100644 --- a/src/earthkit/data/core/plugins.py +++ b/src/earthkit/data/core/plugins.py @@ -18,8 +18,7 @@ import sys from collections import defaultdict from importlib import import_module -from typing import List -from typing import Union +from typing import List, Union import entrypoints diff --git a/src/earthkit/data/core/select.py b/src/earthkit/data/core/select.py index d45459bd3..16be9c3c5 100644 --- a/src/earthkit/data/core/select.py +++ b/src/earthkit/data/core/select.py @@ -61,6 +61,6 @@ def selection_from_index(coord_accessor, kwargs): except IndexError as e: raise IndexError( - (f"Invalid index={v}. Index for key={k} must be in the range of" f"(0, {len(coord_accessor(k))}) {e}") + (f"Invalid index={v}. Index for key={k} must be in the range of (0, {len(coord_accessor(k))}) {e}") ) return _kwargs diff --git a/src/earthkit/data/encoders/__init__.py b/src/earthkit/data/encoders/__init__.py index d0b40c11c..ad4c77676 100644 --- a/src/earthkit/data/encoders/__init__.py +++ b/src/earthkit/data/encoders/__init__.py @@ -9,8 +9,7 @@ import logging import os -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from importlib import import_module LOG = logging.getLogger(__name__) @@ -44,15 +43,15 @@ class EncodedData(metaclass=ABCMeta): @abstractmethod def to_bytes(self): - """Return the data as a bytesarray""" + """Return the data as a bytesarray.""" pass @abstractmethod def to_file(self, f): """Write the data to a file. - Parameters: - ----------- + Parameters + ---------- f: file-like object File-like object to write to """ @@ -68,8 +67,8 @@ class Encoder(metaclass=ABCMeta): An encoder is used to encode data to a specific format that can be used by a :class:`Target`. - Parameters: - ----------- + Parameters + ---------- template: obj, None The template to use to encode the data. Can be overridden in the :obj:`encode` method. metadata: dict, None @@ -96,8 +95,8 @@ def encode( ) -> EncodedData: """Encode the data. - Parameters: - ----------- + Parameters + ---------- data: obj, None The data to encode. Should be used via double dispatch. Must have an ``_encode()`` method, which will call the appropriate ``_encode_*`` method on the :class:`Encoder`. @@ -116,8 +115,8 @@ def encode( **kwargs: dict Additional keyword arguments. - Returns: - -------- + Returns + ------- EncodedData The encoded data. """ @@ -127,48 +126,52 @@ def encode( def _encode(self, data, **kwargs) -> EncodedData: """Subclass implementation of the encoding logic. - Parameters: - ----------- + Parameters + ---------- field: :obj:`Base` The data to encode - Double dispatch method that called from a ``data`` to encode itself.""" + Double dispatch method that called from a ``data`` to encode itself. + """ pass @abstractmethod def _encode_field(self, field, **kwargs) -> EncodedData: """Subclass implementation of the encoding logic for a Field. - Parameters: - ----------- + Parameters + ---------- field: :obj:`Field` The Field to encode - Double dispatch method that called from ``field`` to encode itself.""" + Double dispatch method that called from ``field`` to encode itself. + """ pass @abstractmethod def _encode_fieldlist(self, fieldlist, **kwargs) -> EncodedData: """Subclass implementation of the encoding logic for a FieldList. - Parameters: - ----------- + Parameters + ---------- fieldlist: :obj:`FieldList` The FieldList to encode - Double dispatch method that called from ``fieldlist`` to encode itself.""" + Double dispatch method that called from ``fieldlist`` to encode itself. + """ pass @abstractmethod def _encode_xarray(self, data, **kwargs) -> EncodedData: """Subclass implementation of the encoding logic for Xarray data. - Parameters: - ----------- + Parameters + ---------- data: Xarray DataArray or Dataset The data to encode - Double dispatch method that called from ``data`` to encode itself.""" + Double dispatch method that called from ``data`` to encode itself. + """ pass diff --git a/src/earthkit/data/encoders/csv.py b/src/earthkit/data/encoders/csv.py index 2432dfb19..68c9a5be2 100644 --- a/src/earthkit/data/encoders/csv.py +++ b/src/earthkit/data/encoders/csv.py @@ -9,8 +9,7 @@ import logging -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/encoders/geotiff.py b/src/earthkit/data/encoders/geotiff.py index 4820873e2..7fb2a2058 100644 --- a/src/earthkit/data/encoders/geotiff.py +++ b/src/earthkit/data/encoders/geotiff.py @@ -9,8 +9,7 @@ import logging -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/encoders/grib.py b/src/earthkit/data/encoders/grib.py index 5292cb0a0..d42ccff73 100644 --- a/src/earthkit/data/encoders/grib.py +++ b/src/earthkit/data/encoders/grib.py @@ -11,12 +11,10 @@ import logging from functools import lru_cache -from earthkit.data.decorators import normalize -from earthkit.data.decorators import normalize_grib_keys +from earthkit.data.decorators import normalize, normalize_grib_keys from earthkit.data.utils.humanize import list_to_human -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) @@ -101,7 +99,7 @@ def _gg_pl(N): class GribHandleMaker: - """Create a new GribCodesHandle from a template, field or metadata""" + """Create a new GribCodesHandle from a template, field or metadata.""" def __init__(self, template=None): self.template = template @@ -109,7 +107,7 @@ def __init__(self, template=None): def make(self, field=None, values=None, metadata=None, template=None): """Create a new GribCodesHandle from a template, field or metadata - May modify existing metadata + May modify existing metadata. Parameters ---------- @@ -306,12 +304,11 @@ def _gg_field(self, values, metadata): if octahedral or levtype == "sfc": return f"reduced_gg_{levtype}_grib{edition}" else: - return f"reduced_gg_{levtype}_{N}_grib{edition}" class GribEncoder(Encoder): - """Encode GRIB data""" + """Encode GRIB data.""" def __init__(self, **kwargs): super().__init__(**kwargs) @@ -338,7 +335,6 @@ def encode( """ Parameters ---------- - data: Field The data to encode values: numpy.ndarray diff --git a/src/earthkit/data/encoders/internal_pass_through.py b/src/earthkit/data/encoders/internal_pass_through.py index 8ab0aed99..5bf06fc5b 100644 --- a/src/earthkit/data/encoders/internal_pass_through.py +++ b/src/earthkit/data/encoders/internal_pass_through.py @@ -9,8 +9,7 @@ import logging -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) @@ -38,7 +37,7 @@ def metadata(self, key): class _InternalPassThroughEncoder(Encoder): - """A pass-through encoder implemented for Readers""" + """A pass-through encoder implemented for Readers.""" def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/src/earthkit/data/encoders/netcdf.py b/src/earthkit/data/encoders/netcdf.py index 01a4b73c0..46db717d6 100644 --- a/src/earthkit/data/encoders/netcdf.py +++ b/src/earthkit/data/encoders/netcdf.py @@ -9,8 +9,7 @@ import logging -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/encoders/zarr.py b/src/earthkit/data/encoders/zarr.py index 101bdf355..b243f2082 100644 --- a/src/earthkit/data/encoders/zarr.py +++ b/src/earthkit/data/encoders/zarr.py @@ -9,8 +9,7 @@ import logging -from . import EncodedData -from . import Encoder +from . import EncodedData, Encoder LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/indexing/database/sql.py b/src/earthkit/data/indexing/database/sql.py index e2402166c..74f3d6594 100644 --- a/src/earthkit/data/indexing/database/sql.py +++ b/src/earthkit/data/indexing/database/sql.py @@ -22,14 +22,16 @@ from earthkit.data.indexing.database.json import json_serialiser from earthkit.data.utils.parts import Part -from . import FILEPARTS_KEY_NAMES -from . import MORE_KEY_NAMES -from . import MORE_KEY_NAMES_WITH_UNDERSCORE -from . import STATISTICS_KEY_NAMES -from . import Database -from . import FloatDBKey -from . import IntDBKey -from . import StrDBKey +from . import ( + FILEPARTS_KEY_NAMES, + MORE_KEY_NAMES, + MORE_KEY_NAMES_WITH_UNDERSCORE, + STATISTICS_KEY_NAMES, + Database, + FloatDBKey, + IntDBKey, + StrDBKey, +) LOG = logging.getLogger(__name__) @@ -332,7 +334,7 @@ def join(self, lst): return None assert new_view != old_view - return f"CREATE TEMP VIEW IF NOT EXISTS {new_view} AS SELECT *, {select} " f"FROM {old_view};" + return f"CREATE TEMP VIEW IF NOT EXISTS {new_view} AS SELECT *, {select} FROM {old_view};" class SqlOrder(SqlFilter): @@ -466,7 +468,7 @@ def __init__( self.dbkeys = EntriesLoader(self.connection).keys def __str__(self): - return f"{self.__class__.__name__}({self.db_path}," f"(filters=[{','.join([str(_) for _ in self._filters])}])" + return f"{self.__class__.__name__}({self.db_path}, filters=[{','.join([str(_) for _ in self._filters])}])" def build_indexes(self): EntriesLoader(self.connection).build_sql_indexes() @@ -488,7 +490,7 @@ def connection(self): def unique_values(self, *coords, remapping=None, progress_bar=True): """Given a list of metadata attributes, such as date, param, levels, - returns the list of unique values for each attributes + returns the list of unique values for each attributes. """ remapping = build_remapping(remapping) with self.connection as con: diff --git a/src/earthkit/data/indexing/fieldlist.py b/src/earthkit/data/indexing/fieldlist.py index 8dc77f6e4..f3314009b 100644 --- a/src/earthkit/data/indexing/fieldlist.py +++ b/src/earthkit/data/indexing/fieldlist.py @@ -9,8 +9,7 @@ from earthkit.data.core.fieldlist import FieldList -from earthkit.data.core.metadata import Metadata -from earthkit.data.core.metadata import WrappedMetadata +from earthkit.data.core.metadata import Metadata, WrappedMetadata class SimpleFieldList(FieldList): @@ -149,7 +148,7 @@ def handle(self): return self._metadata._handle def _encode(self, encoder, **kwargs): - """Double dispatch to the encoder""" + """Double dispatch to the encoder.""" md = {} # wrapped metadata if hasattr(self._metadata, "extra"): diff --git a/src/earthkit/data/indexing/tensor.py b/src/earthkit/data/indexing/tensor.py index df31a822d..d1fe4e13f 100644 --- a/src/earthkit/data/indexing/tensor.py +++ b/src/earthkit/data/indexing/tensor.py @@ -10,21 +10,18 @@ import functools import itertools import logging -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod import numpy as np - -from earthkit.data.core.index import Selection -from earthkit.data.core.index import normalize_selection from earthkit.utils.array import array_namespace as eku_array_namespace +from earthkit.data.core.index import Selection, normalize_selection + LOG = logging.getLogger(__name__) def coords_to_index(coords, shape) -> int: - """ - Map user coords to field index""" + """Map user coords to field index.""" index = 0 n = 1 for i in range(len(coords) - 1, -1, -1): @@ -256,7 +253,7 @@ def _index_to_coords_value(index, tensor): def _check(self): if self._full_shape != self._user_shape + self._field_shape: raise ValueError( - (f"shape={self._full_shape} differs from expected shape=" f"{self._user_shape} + {self._field_shape}") + (f"shape={self._full_shape} differs from expected shape={self._user_shape} + {self._field_shape}") ) shape = self._coords_shape(self._user_coords) + self._dims_shape(self._field_dims) diff --git a/src/earthkit/data/mirrors/directory_mirror.py b/src/earthkit/data/mirrors/directory_mirror.py index d5a66c243..7e9be9464 100644 --- a/src/earthkit/data/mirrors/directory_mirror.py +++ b/src/earthkit/data/mirrors/directory_mirror.py @@ -14,8 +14,7 @@ from earthkit.data.sources.file import FileSource from earthkit.data.sources.url import Url -from . import BaseMirror -from . import MirrorConnection +from . import BaseMirror, MirrorConnection LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/readers/__init__.py b/src/earthkit/data/readers/__init__.py index 0367affd8..22249ab5a 100644 --- a/src/earthkit/data/readers/__init__.py +++ b/src/earthkit/data/readers/__init__.py @@ -16,8 +16,7 @@ from earthkit.data.core import Base from earthkit.data.core.config import CONFIG -from earthkit.data.decorators import detect_out_filename -from earthkit.data.decorators import locked +from earthkit.data.decorators import detect_out_filename, locked LOG = logging.getLogger(__name__) @@ -167,9 +166,7 @@ def _find_reader(method_name, source, path_or_data, **kwargs): def _unknown(method_name, source, path_or_data, **kwargs): - from .unknown import UnknownMemoryReader - from .unknown import UnknownReader - from .unknown import UnknownStreamReader + from .unknown import UnknownMemoryReader, UnknownReader, UnknownStreamReader unknowns = { "reader": UnknownReader, @@ -190,7 +187,7 @@ def _empty(source, path, **kwargs): def reader(source, path, **kwargs): - """Create the reader for a file/directory specified by path""" + """Create the reader for a file/directory specified by path.""" assert isinstance(path, str), source if hasattr(source, "reader"): @@ -236,7 +233,7 @@ def reader(source, path, **kwargs): def memory_reader(source, buffer, **kwargs): - """Create a reader for data held in a memory buffer""" + """Create a reader for data held in a memory buffer.""" assert isinstance(buffer, (bytes, bytearray)), source n_bytes = CONFIG.get("reader-type-check-bytes") magic = buffer[: min(n_bytes, len(buffer) - 1)] @@ -245,7 +242,7 @@ def memory_reader(source, buffer, **kwargs): def stream_reader(source, stream, memory, **kwargs): - """Create a reader for a stream""" + """Create a reader for a stream.""" magic = None if hasattr(stream, "peek") and callable(stream.peek): try: diff --git a/src/earthkit/data/readers/bufr/bufr.py b/src/earthkit/data/readers/bufr/bufr.py index 6de4046d9..2d52387d5 100644 --- a/src/earthkit/data/readers/bufr/bufr.py +++ b/src/earthkit/data/readers/bufr/bufr.py @@ -14,12 +14,8 @@ from pdbufr.high_level_bufr.bufr import bufr_code_is_coord from earthkit.data.core import Base -from earthkit.data.core.index import Index -from earthkit.data.core.index import MaskIndex -from earthkit.data.core.index import MultiIndex -from earthkit.data.utils.message import CodesHandle -from earthkit.data.utils.message import CodesMessagePositionIndex -from earthkit.data.utils.message import CodesReader +from earthkit.data.core.index import Index, MaskIndex, MultiIndex +from earthkit.data.utils.message import CodesHandle, CodesMessagePositionIndex, CodesReader from earthkit.data.utils.metadata.args import metadata_argument from earthkit.data.utils.parts import Part from earthkit.data.utils.summary import make_bufr_html_tree @@ -87,13 +83,13 @@ def __init__(self, handle, path, offset): self._unpacked = False def unpack(self): - """Decode data section""" + """Decode data section.""" if not self._unpacked: eccodes.codes_set(self._handle, "unpack", 1) self._unpacked = True def pack(self): - """Encode data section""" + """Encode data section.""" if self._unpacked: eccodes.codes_set(self._handle, "pack", 1) self._unpacked = False @@ -128,7 +124,7 @@ def __next__(self): return _KeyIterator(self._handle) def keys(self, namespace=None): - """Iterate over all the available keys""" + """Iterate over all the available keys.""" return self.__iter__() def as_namespace(self, namespace=None): @@ -198,7 +194,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self._handle = None def __setitem__(self, key, value): - """Sets value associated with ``key``""" + """Sets value associated with ``key``.""" if isinstance(value, list): return eccodes.codes_set_array(self.handle._handle, key, value) else: @@ -300,7 +296,7 @@ def metadata(self, *keys, astype=None, **kwargs): return self.handle.as_namespace() def is_coord(self, key): - """Check if the specified key is a BUFR coordinate descriptor + """Check if the specified key is a BUFR coordinate descriptor. Parameters ---------- @@ -666,7 +662,7 @@ def number_of_parts(self): class BUFRReader(BUFRInOneFile, Reader): - """Represents a BUFR file""" + """Represents a BUFR file.""" appendable = True # BUFR messages can be added to the same file diff --git a/src/earthkit/data/readers/csv.py b/src/earthkit/data/readers/csv.py index 3ab46e784..a0cb2a411 100644 --- a/src/earthkit/data/readers/csv.py +++ b/src/earthkit/data/readers/csv.py @@ -127,7 +127,7 @@ def is_csv(path, probe_size=4096, compression=None): class CSVReader(Reader): - r"""Class representing CSV data""" + r"""Class representing CSV data.""" def __init__(self, source, path, compression=None): super().__init__(source, path) diff --git a/src/earthkit/data/readers/geojson.py b/src/earthkit/data/readers/geojson.py index 906d04880..89306d0fb 100644 --- a/src/earthkit/data/readers/geojson.py +++ b/src/earthkit/data/readers/geojson.py @@ -23,7 +23,7 @@ def __init__(self, source, path): self.fields = None def _scan(self): - """For geojson, a field is a feature""" + """For geojson, a field is a feature.""" if self.fields is None: self.fields = self.get_fields() @@ -32,12 +32,12 @@ def __repr__(self): # return self.to_pandas().__repr__() def _repr_html_(self): - html_repr = f"

GeojsonReader(represented as a geopandas object):

" f"{self.to_pandas()._repr_html_()}" + html_repr = f"

GeojsonReader(represented as a geopandas object):

{self.to_pandas()._repr_html_()}" return html_repr def __iter__(self): - """Iterate over features in geojson via pandas""" + """Iterate over features in geojson via pandas.""" self._scan() return iter(self.fields) diff --git a/src/earthkit/data/readers/geotiff.py b/src/earthkit/data/readers/geotiff.py index 83ea30c8a..bb47ef679 100644 --- a/src/earthkit/data/readers/geotiff.py +++ b/src/earthkit/data/readers/geotiff.py @@ -10,8 +10,7 @@ import numpy as np -from earthkit.data.core.fieldlist import Field -from earthkit.data.core.fieldlist import FieldList +from earthkit.data.core.fieldlist import Field, FieldList from earthkit.data.core.geography import Geography from earthkit.data.core.metadata import RawMetadata from earthkit.data.decorators import thread_safe_cached_property @@ -21,7 +20,6 @@ class GeoTIFFGeography(Geography): - def __init__(self, ds): self._ds = ds self.x_dim = ds.rio.x_dim @@ -97,7 +95,6 @@ def mars_area(self): class GeoTIFFMetadata(RawMetadata): - LS_KEYS = ["variable", "band"] def __init__(self, field, band, geography=None): @@ -116,7 +113,7 @@ def ls_keys(self): class GeoTIFFField(Field): - """A GeoTIFF band""" + """A GeoTIFF band.""" def __init__(self, da, band, geography=None): super().__init__() @@ -148,7 +145,7 @@ def write(self, f): class GeoTIFFFieldList(FieldList): - """A list of GeoTIFF bands""" + """A list of GeoTIFF bands.""" FIELD_TYPE = GeoTIFFField diff --git a/src/earthkit/data/readers/grib/__init__.py b/src/earthkit/data/readers/grib/__init__.py index 42588268c..e32466f10 100644 --- a/src/earthkit/data/readers/grib/__init__.py +++ b/src/earthkit/data/readers/grib/__init__.py @@ -50,8 +50,7 @@ def memory_reader(source, buffer, *, magic=None, deeper_check=False, **kwargs): if _match_magic(magic, deeper_check): import io - from .memory import GribFieldListInMemory - from .memory import GribStreamReader + from .memory import GribFieldListInMemory, GribStreamReader # convert the memory buffer into a stream and use the stream reader. See # https://github.com/ecmwf/earthkit-data/issues/740 for details @@ -75,8 +74,7 @@ def stream_reader( **kwargs, ): if _is_default(magic, content_type) or _match_magic(magic, deeper_check): - from .memory import GribFieldListInMemory - from .memory import GribStreamReader + from .memory import GribFieldListInMemory, GribStreamReader r = GribStreamReader(stream, **kwargs) if memory: diff --git a/src/earthkit/data/readers/grib/codes.py b/src/earthkit/data/readers/grib/codes.py index 331a35a2f..b38253980 100644 --- a/src/earthkit/data/readers/grib/codes.py +++ b/src/earthkit/data/readers/grib/codes.py @@ -12,15 +12,13 @@ import eccodes import numpy as np +from earthkit.utils.array.convert import convert_dtype from earthkit.data.core.fieldlist import Field from earthkit.data.decorators import thread_safe_cached_property from earthkit.data.indexing.fieldlist import ClonedFieldCore from earthkit.data.readers.grib.metadata import GribFieldMetadata -from earthkit.data.utils.message import CodesHandle -from earthkit.data.utils.message import CodesMessagePositionIndex -from earthkit.data.utils.message import CodesReader -from earthkit.utils.array.convert import convert_dtype +from earthkit.data.utils.message import CodesHandle, CodesMessagePositionIndex, CodesReader LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/readers/grib/gridspec.py b/src/earthkit/data/readers/grib/gridspec.py index d2764be72..d82e95a7f 100644 --- a/src/earthkit/data/readers/grib/gridspec.py +++ b/src/earthkit/data/readers/grib/gridspec.py @@ -9,8 +9,7 @@ import logging import re -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from functools import lru_cache from earthkit.data.core.gridspec import GridSpec diff --git a/src/earthkit/data/readers/grib/index/__init__.py b/src/earthkit/data/readers/grib/index/__init__.py index d946afbab..a50f6299d 100644 --- a/src/earthkit/data/readers/grib/index/__init__.py +++ b/src/earthkit/data/readers/grib/index/__init__.py @@ -15,14 +15,14 @@ from collections import defaultdict from earthkit.data.core.fieldlist import FieldList -from earthkit.data.core.index import MaskIndex -from earthkit.data.core.index import MultiIndex -from earthkit.data.decorators import alias_argument -from earthkit.data.decorators import detect_out_filename -from earthkit.data.indexing.database import FILEPARTS_KEY_NAMES -from earthkit.data.indexing.database import MORE_KEY_NAMES -from earthkit.data.indexing.database import MORE_KEY_NAMES_WITH_UNDERSCORE -from earthkit.data.indexing.database import STATISTICS_KEY_NAMES +from earthkit.data.core.index import MaskIndex, MultiIndex +from earthkit.data.decorators import alias_argument, detect_out_filename +from earthkit.data.indexing.database import ( + FILEPARTS_KEY_NAMES, + MORE_KEY_NAMES, + MORE_KEY_NAMES_WITH_UNDERSCORE, + STATISTICS_KEY_NAMES, +) from earthkit.data.readers.grib.codes import GribField from earthkit.data.readers.grib.pandas import PandasMixIn from earthkit.data.readers.grib.xarray import XarrayMixIn @@ -398,7 +398,7 @@ def __len__(self): return self.number_of_parts() def _cache_diag(self): - """For testing only""" + """For testing only.""" r = defaultdict(int) r.update(self._field_manager.diag()) r.update(self._handle_manager.diag()) diff --git a/src/earthkit/data/readers/grib/index/db.py b/src/earthkit/data/readers/grib/index/db.py index 64d49d776..2c80cda7b 100644 --- a/src/earthkit/data/readers/grib/index/db.py +++ b/src/earthkit/data/readers/grib/index/db.py @@ -72,7 +72,7 @@ def load(target, *args): @classmethod def from_url(cls, url, patch_entry=None, **kwargs): - """Create a database from a given url""" + """Create a database from a given url.""" import requests from multiurl import robust diff --git a/src/earthkit/data/readers/grib/index/sql.py b/src/earthkit/data/readers/grib/index/sql.py index d86c81fd6..4087d7c09 100644 --- a/src/earthkit/data/readers/grib/index/sql.py +++ b/src/earthkit/data/readers/grib/index/sql.py @@ -11,15 +11,10 @@ from collections import namedtuple from earthkit.data.core.constants import DATETIME -from earthkit.data.core.order import build_remapping -from earthkit.data.core.order import normalize_order_by +from earthkit.data.core.order import build_remapping, normalize_order_by from earthkit.data.core.select import normalize_selection -from earthkit.data.decorators import cached_method -from earthkit.data.decorators import normalize -from earthkit.data.indexing.database.sql import SqlDatabase -from earthkit.data.indexing.database.sql import SqlOrder -from earthkit.data.indexing.database.sql import SqlRemapping -from earthkit.data.indexing.database.sql import SqlSelection +from earthkit.data.decorators import cached_method, normalize +from earthkit.data.indexing.database.sql import SqlDatabase, SqlOrder, SqlRemapping, SqlSelection from earthkit.data.readers.grib.index.db import FieldListInFilesWithDBIndex from earthkit.data.utils.serialise import register_serialisation @@ -60,7 +55,7 @@ def _find_all_indices_dict(self): def unique_values(self, *coords, remapping=None, progress_bar=None): """Given a list of metadata attributes, such as date, param, levels, - returns the list of unique values for each attributes + returns the list of unique values for each attributes. """ keys = coords diff --git a/src/earthkit/data/readers/grib/memory.py b/src/earthkit/data/readers/grib/memory.py index 758b3435b..9b82224d8 100644 --- a/src/earthkit/data/readers/grib/memory.py +++ b/src/earthkit/data/readers/grib/memory.py @@ -11,11 +11,9 @@ import eccodes -from earthkit.data.indexing.fieldlist import ClonedFieldCore -from earthkit.data.indexing.fieldlist import SimpleFieldList +from earthkit.data.indexing.fieldlist import ClonedFieldCore, SimpleFieldList from earthkit.data.readers import Reader -from earthkit.data.readers.grib.codes import GribCodesHandle -from earthkit.data.readers.grib.codes import GribField +from earthkit.data.readers.grib.codes import GribCodesHandle, GribField LOG = logging.getLogger(__name__) @@ -113,7 +111,7 @@ class GribStreamReader(GribMemoryReader): """Wrapper around eccodes.Streamreader. The problem is that when iterating via the StreamReader it returns an eccodes.GRIBMessage that releases the handle when deleted. However, the handle has to be managed by earthkit-data so we access it directly - using _next_handle + using _next_handle. """ def __init__(self, stream, **kwargs): @@ -142,7 +140,7 @@ def mutate_source(self): class GribFieldInMemory(GribField): - """Represents a GRIB message in memory""" + """Represents a GRIB message in memory.""" def __init__(self, handle, use_metadata_cache=False): super().__init__(None, None, None, use_metadata_cache=use_metadata_cache) @@ -194,7 +192,7 @@ def __init__(self, field, **kwargs): class GribFieldListInMemory(SimpleFieldList): - """Represent a GRIB field list in memory loaded lazily""" + """Represent a GRIB field list in memory loaded lazily.""" def __init__(self, source, reader, *args, **kwargs): """The reader must support __next__.""" diff --git a/src/earthkit/data/readers/grib/metadata.py b/src/earthkit/data/readers/grib/metadata.py index c8f5f2b0b..4cd9c56b7 100644 --- a/src/earthkit/data/readers/grib/metadata.py +++ b/src/earthkit/data/readers/grib/metadata.py @@ -12,18 +12,15 @@ import warnings from abc import abstractmethod +from earthkit.utils.array import array_namespace + from earthkit.data.core.geography import Geography -from earthkit.data.core.metadata import Metadata -from earthkit.data.core.metadata import MetadataAccessor -from earthkit.data.core.metadata import MetadataCacheHandler -from earthkit.data.core.metadata import WrappedMetadata +from earthkit.data.core.metadata import Metadata, MetadataAccessor, MetadataCacheHandler, WrappedMetadata from earthkit.data.decorators import thread_safe_cached_property from earthkit.data.indexing.database import GRIB_KEYS_NAMES from earthkit.data.readers.grib.gridspec import make_gridspec from earthkit.data.utils.bbox import BoundingBox -from earthkit.data.utils.dates import datetime_from_grib -from earthkit.data.utils.dates import to_timedelta -from earthkit.utils.array import array_namespace +from earthkit.data.utils.dates import datetime_from_grib, to_timedelta LOG = logging.getLogger(__name__) @@ -653,7 +650,7 @@ def __init__(self, cache=None, **kwargs): @staticmethod def _handle_type(): """Return the expected handle type. Implemented like this - to avoid cyclic import + to avoid cyclic import. """ if GribMetadata.__handle_type is None: from earthkit.data.readers.grib.codes import GribCodesHandle @@ -957,13 +954,11 @@ def dump(self, namespace=all, **kwargs): for ns in namespace: v = self.as_namespace(ns) if v: - r.append( - { - "title": ns if ns else "default", - "data": v, - "tooltip": f"Keys in the ecCodes {ns} namespace", - } - ) + r.append({ + "title": ns if ns else "default", + "data": v, + "tooltip": f"Keys in the ecCodes {ns} namespace", + }) return format_namespace_dump(r, selected="parameter", details=self.__class__.__name__, **kwargs) diff --git a/src/earthkit/data/readers/grib/output.py b/src/earthkit/data/readers/grib/output.py index 2e84df148..10c8fa31e 100644 --- a/src/earthkit/data/readers/grib/output.py +++ b/src/earthkit/data/readers/grib/output.py @@ -15,8 +15,7 @@ import deprecation -from earthkit.data.decorators import normalize -from earthkit.data.decorators import normalize_grib_keys +from earthkit.data.decorators import normalize, normalize_grib_keys from earthkit.data.utils.humanize import list_to_human LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/readers/grib/parsing.py b/src/earthkit/data/readers/grib/parsing.py index 4ee95bc96..248d0f533 100644 --- a/src/earthkit/data/readers/grib/parsing.py +++ b/src/earthkit/data/readers/grib/parsing.py @@ -144,7 +144,7 @@ def _index_path(path): class PathParserIterator: """Delays parsing the directory for the list of files - until the iterator is actually used (calling __iter__) + until the iterator is actually used (calling __iter__). """ def __init__( diff --git a/src/earthkit/data/readers/grib/virtual.py b/src/earthkit/data/readers/grib/virtual.py index ab86422c5..1da3fe2e5 100644 --- a/src/earthkit/data/readers/grib/virtual.py +++ b/src/earthkit/data/readers/grib/virtual.py @@ -12,10 +12,7 @@ from earthkit.data.core.fieldlist import Field from earthkit.data.core.metadata import WrappedMetadata from earthkit.data.decorators import thread_safe_cached_property -from earthkit.data.utils.dates import date_to_grib -from earthkit.data.utils.dates import datetime_from_grib -from earthkit.data.utils.dates import time_to_grib -from earthkit.data.utils.dates import to_timedelta +from earthkit.data.utils.dates import date_to_grib, datetime_from_grib, time_to_grib, to_timedelta from .index import GribFieldList diff --git a/src/earthkit/data/readers/grib/xarray.py b/src/earthkit/data/readers/grib/xarray.py index 689d1a503..5b13f07a0 100644 --- a/src/earthkit/data/readers/grib/xarray.py +++ b/src/earthkit/data/readers/grib/xarray.py @@ -12,8 +12,7 @@ from earthkit.data.utils import ensure_dict from earthkit.data.utils.kwargs import Kwargs -from earthkit.data.utils.serialise import deserialise_state -from earthkit.data.utils.serialise import serialise_state +from earthkit.data.utils.serialise import deserialise_state, serialise_state LOG = logging.getLogger(__name__) @@ -54,7 +53,6 @@ def __len__(self): class XarrayMixIn: - @staticmethod def _kwargs_for_xarray(): import inspect diff --git a/src/earthkit/data/readers/netcdf/__init__.py b/src/earthkit/data/readers/netcdf/__init__.py index da4e6689d..8ebecb481 100644 --- a/src/earthkit/data/readers/netcdf/__init__.py +++ b/src/earthkit/data/readers/netcdf/__init__.py @@ -9,8 +9,7 @@ from .. import Reader -from .fieldlist import NetCDFFieldListFromFile -from .fieldlist import NetCDFFieldListFromURL +from .fieldlist import NetCDFFieldListFromFile, NetCDFFieldListFromURL class NetCDFFieldListReader(NetCDFFieldListFromFile, Reader): diff --git a/src/earthkit/data/readers/netcdf/dataset.py b/src/earthkit/data/readers/netcdf/dataset.py index 6dc20e6dd..b25bd3899 100644 --- a/src/earthkit/data/readers/netcdf/dataset.py +++ b/src/earthkit/data/readers/netcdf/dataset.py @@ -235,7 +235,7 @@ def shape(self): class DataSet: - """Class that wraps a xarray dataset to provide caching""" + """Class that wraps a xarray dataset to provide caching.""" def __init__(self, ds): self._ds = ds diff --git a/src/earthkit/data/readers/netcdf/field.py b/src/earthkit/data/readers/netcdf/field.py index 05ca654c0..88fe22242 100644 --- a/src/earthkit/data/readers/netcdf/field.py +++ b/src/earthkit/data/readers/netcdf/field.py @@ -14,15 +14,13 @@ from earthkit.data.core.fieldlist import Field from earthkit.data.core.geography import Geography -from earthkit.data.core.metadata import MetadataAccessor -from earthkit.data.core.metadata import RawMetadata +from earthkit.data.core.metadata import MetadataAccessor, RawMetadata from earthkit.data.decorators import thread_safe_cached_property from earthkit.data.indexing.fieldlist import ClonedFieldCore from earthkit.data.utils.bbox import BoundingBox from earthkit.data.utils.dates import to_datetime -from .coords import LevelSlice -from .coords import TimeSlice +from .coords import LevelSlice, TimeSlice LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/readers/netcdf/fieldlist.py b/src/earthkit/data/readers/netcdf/fieldlist.py index 966b08799..90905941e 100644 --- a/src/earthkit/data/readers/netcdf/fieldlist.py +++ b/src/earthkit/data/readers/netcdf/fieldlist.py @@ -13,17 +13,12 @@ import deprecation from earthkit.data.core.fieldlist import FieldList -from earthkit.data.core.index import MaskIndex -from earthkit.data.core.index import MultiIndex +from earthkit.data.core.index import MaskIndex, MultiIndex from earthkit.data.decorators import thread_safe_cached_property -from .coords import LevelCoordinate -from .coords import OtherCoordinate -from .coords import TimeCoordinate -from .dataset import GEOGRAPHIC_COORDS -from .dataset import DataSet -from .field import NetCDFField -from .field import XArrayField +from .coords import LevelCoordinate, OtherCoordinate, TimeCoordinate +from .dataset import GEOGRAPHIC_COORDS, DataSet +from .field import NetCDFField, XArrayField LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/readers/shapefile.py b/src/earthkit/data/readers/shapefile.py index 15faf2a73..04433131b 100644 --- a/src/earthkit/data/readers/shapefile.py +++ b/src/earthkit/data/readers/shapefile.py @@ -24,8 +24,7 @@ def __repr__(self): def _repr_html_(self): html_repr = ( - f"

{self.__class__.__name__}(represented as a geopandas object):

" - f"{self.to_pandas()._repr_html_()}" + f"

{self.__class__.__name__}(represented as a geopandas object):

{self.to_pandas()._repr_html_()}" ) return html_repr diff --git a/src/earthkit/data/readers/text.py b/src/earthkit/data/readers/text.py index 512d8a2ea..e7f77136c 100644 --- a/src/earthkit/data/readers/text.py +++ b/src/earthkit/data/readers/text.py @@ -9,8 +9,7 @@ from . import Reader -from .csv import CSVReader -from .csv import is_csv +from .csv import CSVReader, is_csv def is_text(path, prob_lines=1000, probe_size=4096): diff --git a/src/earthkit/data/readers/unknown.py b/src/earthkit/data/readers/unknown.py index 70da87a3f..a0f00c6f8 100644 --- a/src/earthkit/data/readers/unknown.py +++ b/src/earthkit/data/readers/unknown.py @@ -54,10 +54,7 @@ def __init__(self, source, data, **kwargs): super().__init__(source, **kwargs) if not self.skip_warning: LOG.warning( - ( - f"Unknown stream data type, no reader available. " - f"magic={self.magic} content_type={self.content_type}" - ) + (f"Unknown stream data type, no reader available. magic={self.magic} content_type={self.content_type}") ) @@ -66,8 +63,5 @@ def __init__(self, source, data, **kwargs): super().__init__(source, **kwargs) if not self.skip_warning: LOG.warning( - ( - f"Unknown memory data type, no reader available. " - f"magic={self.magic} content_type={self.content_type}" - ) + (f"Unknown memory data type, no reader available. magic={self.magic} content_type={self.content_type}") ) diff --git a/src/earthkit/data/readers/zarr.py b/src/earthkit/data/readers/zarr.py index 97cb56183..8889c0b6a 100644 --- a/src/earthkit/data/readers/zarr.py +++ b/src/earthkit/data/readers/zarr.py @@ -14,7 +14,6 @@ class ZarrReader(XArrayFieldList, Reader): - def __init__(self, source, path, **kwargs): Reader.__init__(self, source, path, **kwargs) XArrayFieldList.__init__(self, self._open_zarr(**kwargs)) diff --git a/src/earthkit/data/readers/zip.py b/src/earthkit/data/readers/zip.py index 6e369b1f2..b888ba478 100644 --- a/src/earthkit/data/readers/zip.py +++ b/src/earthkit/data/readers/zip.py @@ -18,7 +18,7 @@ class InfoWrapper: - """A class so that ZipInfo has the same interface as TarInfo""" + """A class so that ZipInfo has the same interface as TarInfo.""" def __init__(self, member): self.member = member diff --git a/src/earthkit/data/sources/__init__.py b/src/earthkit/data/sources/__init__.py index bd47ec870..4bd7462b7 100644 --- a/src/earthkit/data/sources/__init__.py +++ b/src/earthkit/data/sources/__init__.py @@ -20,7 +20,7 @@ class Source(Base): - """Doc""" + """Doc.""" name = None home_page = "-" diff --git a/src/earthkit/data/sources/array_list.py b/src/earthkit/data/sources/array_list.py index 4b9758261..754233fbf 100644 --- a/src/earthkit/data/sources/array_list.py +++ b/src/earthkit/data/sources/array_list.py @@ -10,9 +10,10 @@ import logging import math +from earthkit.utils.array import array_namespace as eku_array_namespace + from earthkit.data.core.fieldlist import Field from earthkit.data.indexing.fieldlist import ClonedFieldCore -from earthkit.utils.array import array_namespace as eku_array_namespace LOG = logging.getLogger(__name__) @@ -51,7 +52,7 @@ def _array(self): return self._array_ def _values(self, dtype=None): - """Native array type""" + """Native array type.""" if dtype is None: return self._array else: @@ -79,7 +80,7 @@ def __repr__(self): ) def _encode(self, encoder, **kwargs): - """Double dispatch to the encoder""" + """Double dispatch to the encoder.""" values = kwargs.pop("values", None) if values is None: values = self.to_numpy(flatten=True) @@ -144,16 +145,11 @@ def _shape_match(shape1, shape2): array = array_ns.stack([array]) else: raise ValueError( - ( - f"first array dimension={array.shape[0]} differs " - f"from number of metadata objects={len(metadata)}" - ) + (f"first array dimension={array.shape[0]} differs from number of metadata objects={len(metadata)}") ) else: if len(array) != len(metadata): - raise ValueError( - (f"array len=({len(array)}) differs " f"from number of metadata objects=({len(metadata)})") - ) + raise ValueError((f"array len=({len(array)}) differs from number of metadata objects=({len(metadata)})")) fields = [] for i, a in enumerate(array): diff --git a/src/earthkit/data/sources/cds.py b/src/earthkit/data/sources/cds.py index 98f2b79e5..e51294d7a 100644 --- a/src/earthkit/data/sources/cds.py +++ b/src/earthkit/data/sources/cds.py @@ -18,8 +18,7 @@ import yaml from earthkit.data.decorators import normalize -from earthkit.data.utils.request import FileRequestRetriever -from earthkit.data.utils.request import RequestBuilder +from earthkit.data.utils.request import FileRequestRetriever, RequestBuilder from .file import FileSource from .prompt import APIKeyPrompt diff --git a/src/earthkit/data/sources/ecmwf_api.py b/src/earthkit/data/sources/ecmwf_api.py index 2ca87e823..04c393295 100644 --- a/src/earthkit/data/sources/ecmwf_api.py +++ b/src/earthkit/data/sources/ecmwf_api.py @@ -10,8 +10,7 @@ import logging from earthkit.data.decorators import normalize -from earthkit.data.utils.request import FileRequestRetriever -from earthkit.data.utils.request import RequestBuilder +from earthkit.data.utils.request import FileRequestRetriever, RequestBuilder from .file import FileSource from .prompt import APIKeyPrompt diff --git a/src/earthkit/data/sources/ecmwf_open_data.py b/src/earthkit/data/sources/ecmwf_open_data.py index 96e53b027..1b59c0736 100644 --- a/src/earthkit/data/sources/ecmwf_open_data.py +++ b/src/earthkit/data/sources/ecmwf_open_data.py @@ -13,8 +13,7 @@ except ImportError: raise ImportError("ECMWF Open Data access requires 'ecmwf-opendata' to be installed") -from earthkit.data.utils.request import FileRequestRetriever -from earthkit.data.utils.request import RequestBuilder +from earthkit.data.utils.request import FileRequestRetriever, RequestBuilder from .file import FileSource diff --git a/src/earthkit/data/sources/experimental/_iris.py b/src/earthkit/data/sources/experimental/_iris.py index 326732849..dcce54af9 100644 --- a/src/earthkit/data/sources/experimental/_iris.py +++ b/src/earthkit/data/sources/experimental/_iris.py @@ -12,7 +12,6 @@ class IrisSource(Source): - def __init__(self, path, **kwargs): super().__init__(**kwargs) diff --git a/src/earthkit/data/sources/fdb.py b/src/earthkit/data/sources/fdb.py index 4529b8d49..53ab4df9d 100644 --- a/src/earthkit/data/sources/fdb.py +++ b/src/earthkit/data/sources/fdb.py @@ -18,8 +18,7 @@ from earthkit.data.sources.file import FileSource from earthkit.data.sources.stream import StreamSource -from earthkit.data.utils.request import RequestBuilder -from earthkit.data.utils.request import RequestMapper +from earthkit.data.utils.request import RequestBuilder, RequestMapper from . import Source diff --git a/src/earthkit/data/sources/file.py b/src/earthkit/data/sources/file.py index 6ed17ea59..9450c9990 100644 --- a/src/earthkit/data/sources/file.py +++ b/src/earthkit/data/sources/file.py @@ -301,8 +301,7 @@ def to_stream(self): f = open(self.path, "rb") return f else: - from earthkit.data.utils.stream import FilePartStreamReader - from earthkit.data.utils.stream import RequestIterStreamer + from earthkit.data.utils.stream import FilePartStreamReader, RequestIterStreamer stream = FilePartStreamReader(self.path, self.parts) return RequestIterStreamer(iter(stream)) diff --git a/src/earthkit/data/sources/file_pattern.py b/src/earthkit/data/sources/file_pattern.py index 2d45b6a18..09b898f03 100644 --- a/src/earthkit/data/sources/file_pattern.py +++ b/src/earthkit/data/sources/file_pattern.py @@ -8,18 +8,13 @@ # from typing import Any as TypingAny -from typing import Dict -from typing import Optional -from typing import Tuple -from typing import Union +from typing import Dict, Optional, Tuple, Union -from earthkit.data.sources import Source -from earthkit.data.sources import from_source +from earthkit.data.sources import Source, from_source from earthkit.data.sources.empty import EmptySource from earthkit.data.sources.file import File from earthkit.data.sources.multi import MultiSource -from earthkit.data.utils.patterns import HivePattern -from earthkit.data.utils.patterns import Pattern +from earthkit.data.utils.patterns import HivePattern, Pattern class HiveFilePattern(Source): diff --git a/src/earthkit/data/sources/forcings.py b/src/earthkit/data/sources/forcings.py index 818cde1c1..ee5ab2e59 100644 --- a/src/earthkit/data/sources/forcings.py +++ b/src/earthkit/data/sources/forcings.py @@ -13,13 +13,10 @@ import numpy as np -from earthkit.data.core.fieldlist import Field -from earthkit.data.core.fieldlist import FieldList +from earthkit.data.core.fieldlist import Field, FieldList from earthkit.data.core.index import MaskIndex from earthkit.data.core.metadata import RawMetadata -from earthkit.data.decorators import cached_method -from earthkit.data.decorators import normalize -from earthkit.data.decorators import thread_safe_cached_property +from earthkit.data.decorators import cached_method, normalize, thread_safe_cached_property from earthkit.data.indexing.fieldlist import ClonedFieldCore from earthkit.data.utils.dates import to_datetime @@ -281,7 +278,7 @@ def make_datetime(date, time): return date if date.hour or date.minute: raise ValueError( - (f"Duplicate information about time time={time}," f"and time={date.hour}:{date.minute} from date={date}") + (f"Duplicate information about time time={time}, and time={date.hour}:{date.minute} from date={date}") ) assert date.hour == 0, (date, time) assert date.minute == 0, (date, time) @@ -336,7 +333,7 @@ def find_numbers(source_or_dataset): return self.request["number"] assert hasattr(source_or_dataset, "unique_values"), ( - f"{source_or_dataset} (type '{type(source_or_dataset).__name__}') is" " not a proper source or dataset" + f"{source_or_dataset} (type '{type(source_or_dataset).__name__}') is not a proper source or dataset" ) return source_or_dataset.unique_values("number", patches={"number": {None: 0}}).get("number", 0) @@ -358,7 +355,7 @@ def find_dates(source_or_dataset): assert "date" not in self.request and "time" not in self.request assert hasattr(source_or_dataset, "unique_values"), ( - f"{source_or_dataset} (type '{type(source_or_dataset).__name__}') is" " not a proper source or dataset" + f"{source_or_dataset} (type '{type(source_or_dataset).__name__}') is not a proper source or dataset" ) return source_or_dataset.unique_values("valid_datetime")["valid_datetime"] diff --git a/src/earthkit/data/sources/gribjump.py b/src/earthkit/data/sources/gribjump.py index f18ec6606..5de90b365 100644 --- a/src/earthkit/data/sources/gribjump.py +++ b/src/earthkit/data/sources/gribjump.py @@ -16,8 +16,7 @@ import itertools import os from collections import UserList -from typing import Any -from typing import Optional +from typing import Any, Optional import numpy as np @@ -79,11 +78,9 @@ def split_mars_requests(request: dict[str, Any]) -> list[dict[str, Any]]: "Use Python lists to load from multiple fields." ) elif isinstance(v, list) and len(v) == 0: - raise ValueError(f"Cannot expand dictionary with empty list. " f"Found empty list for key '{k}'.") + raise ValueError(f"Cannot expand dictionary with empty list. Found empty list for key '{k}'.") elif isinstance(v, list) and len({type(v_) for v_ in v}) != 1: - raise TypeError( - f"All list values must share the same type but found types {set(map(type, v))} " f"in {k}={v}" - ) + raise TypeError(f"All list values must share the same type but found types {set(map(type, v))} in {k}={v}") list_keywords = sorted(k for k, v in request.items() if isinstance(v, list)) lists = [request[k] for k in list_keywords] @@ -185,7 +182,7 @@ def build_extraction_request( if sum(opt is not None for opt in (ranges, mask, indices)) != 1: raise ValueError( - "Exactly one of 'ranges', 'mask' or 'indices' must be set. " f"Got {ranges=}, {mask=}, {indices=}" + f"Exactly one of 'ranges', 'mask' or 'indices' must be set. Got {ranges=}, {mask=}, {indices=}" ) if ranges is not None: @@ -201,7 +198,6 @@ def build_extraction_request( class ExtractionRequestCollection(UserList): - @classmethod def from_mars_requests( cls, @@ -224,15 +220,15 @@ def from_mars_requests( The mask for the extraction requests, by default None. indices : Optional[np.ndarray], optional The indices for the extraction requests, by default None. + Returns ------- ExtractionRequestCollection A collection of ExtractionRequest objects created from the MARS requests. """ - if sum(opt is not None for opt in (ranges, mask, indices)) != 1: raise ValueError( - "Exactly one of 'ranges', 'mask' or 'indices' must be set. " f"Got {ranges=}, {mask=}, {indices=}" + f"Exactly one of 'ranges', 'mask' or 'indices' must be set. Got {ranges=}, {mask=}, {indices=}" ) if mask is not None: @@ -371,12 +367,10 @@ def _enrich_metadata_with_coordinates(self, indices: np.ndarray, metadata: UserM grid_latitudes = self._latitudes[indices] grid_longitudes = self._longitudes[indices] - metadata = metadata.override( - { - "latitudes": grid_latitudes, - "longitudes": grid_longitudes, - } - ) + metadata = metadata.override({ + "latitudes": grid_latitudes, + "longitudes": grid_longitudes, + }) return metadata def to_xarray(self, *args, **kwargs): @@ -443,12 +437,11 @@ def __init__( keyword arguments passed to the `pyfdb.FDB` constructor. These arguments are only passed to the FDB when fetching coordinates and is not used by GribJump for the extraction itself. """ - super().__init__(**kwargs) if sum(opt is not None for opt in (ranges, mask, indices)) != 1: raise ValueError( - "Exactly one of 'ranges', 'mask' or 'indices' must be set. " f"Got {ranges=}, {mask=}, {indices=}" + f"Exactly one of 'ranges', 'mask' or 'indices' must be set. Got {ranges=}, {mask=}, {indices=}" ) self._ranges = ranges self._mask = mask diff --git a/src/earthkit/data/sources/mars.py b/src/earthkit/data/sources/mars.py index d90e47399..223694150 100644 --- a/src/earthkit/data/sources/mars.py +++ b/src/earthkit/data/sources/mars.py @@ -14,8 +14,7 @@ from earthkit.data.core.config import CONFIG from earthkit.data.core.temporary import temp_file -from .ecmwf_api import ECMWFApi -from .ecmwf_api import MARSAPIKeyPrompt +from .ecmwf_api import ECMWFApi, MARSAPIKeyPrompt LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/sources/multi.py b/src/earthkit/data/sources/multi.py index 0e3fbabe0..44ff9ca06 100644 --- a/src/earthkit/data/sources/multi.py +++ b/src/earthkit/data/sources/multi.py @@ -13,8 +13,7 @@ import deprecation from earthkit.data.core.thread import SoftThreadPool -from earthkit.data.mergers import make_merger -from earthkit.data.mergers import merge_by_class +from earthkit.data.mergers import make_merger, merge_by_class from earthkit.data.sources.empty import EmptySource from earthkit.data.utils.bbox import BoundingBox diff --git a/src/earthkit/data/sources/opendap.py b/src/earthkit/data/sources/opendap.py index 788f6430b..4cbf4cb09 100644 --- a/src/earthkit/data/sources/opendap.py +++ b/src/earthkit/data/sources/opendap.py @@ -7,8 +7,7 @@ # nor does it submit to any jurisdiction. # -from earthkit.data.readers.netcdf import NetCDFFieldListUrlReader -from earthkit.data.readers.netcdf import NetCDFReader +from earthkit.data.readers.netcdf import NetCDFFieldListUrlReader, NetCDFReader from earthkit.data.sources import Source diff --git a/src/earthkit/data/sources/polytope.py b/src/earthkit/data/sources/polytope.py index 31d11ed75..80b70684d 100644 --- a/src/earthkit/data/sources/polytope.py +++ b/src/earthkit/data/sources/polytope.py @@ -19,8 +19,7 @@ from . import Source from .file import File from .prompt import APIKeyPrompt -from .url import Url -from .url import download_to_target +from .url import Url, download_to_target LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/sources/prompt.py b/src/earthkit/data/sources/prompt.py index d768ff6d6..7f856ab73 100644 --- a/src/earthkit/data/sources/prompt.py +++ b/src/earthkit/data/sources/prompt.py @@ -14,9 +14,7 @@ import stat from getpass import getpass -from earthkit.data.core.ipython import HTML -from earthkit.data.core.ipython import display -from earthkit.data.core.ipython import ipython_active +from earthkit.data.core.ipython import HTML, display, ipython_active from earthkit.data.utils.humanize import list_to_human LOG = logging.getLogger(__name__) @@ -157,7 +155,7 @@ def ask(self, p, method): message = f"Please enter a value for {p.get('title')}" if "default" in p: - message += " or leave empty for the default value " f"{p.get('default')}" + message += f" or leave empty for the default value {p.get('default')}" message += ", then press *ENTER*" if "example" in p: message += f" The value should look like {p.get('example')}" diff --git a/src/earthkit/data/sources/s3.py b/src/earthkit/data/sources/s3.py index c6de23d2c..5d17c6c23 100644 --- a/src/earthkit/data/sources/s3.py +++ b/src/earthkit/data/sources/s3.py @@ -165,7 +165,7 @@ def __repr__(self): class S3Source(FileSource): - """Represent an AWS S3 bucket source""" + """Represent an AWS S3 bucket source.""" def __init__( self, diff --git a/src/earthkit/data/sources/url.py b/src/earthkit/data/sources/url.py index e7b240eb2..e27aee53f 100644 --- a/src/earthkit/data/sources/url.py +++ b/src/earthkit/data/sources/url.py @@ -172,8 +172,7 @@ def __init__( ): super().__init__(filter=filter, merger=merger) - from earthkit.data.utils.url import UrlSpec - from earthkit.data.utils.url import UrlSpecItem + from earthkit.data.utils.url import UrlSpec, UrlSpecItem if isinstance(url, UrlSpecItem): self.url_spec = UrlSpec.from_urls(url) diff --git a/src/earthkit/data/sources/virtual_directory.py b/src/earthkit/data/sources/virtual_directory.py index ff5ee06c7..a6d7b5fe4 100644 --- a/src/earthkit/data/sources/virtual_directory.py +++ b/src/earthkit/data/sources/virtual_directory.py @@ -10,8 +10,9 @@ import warnings from collections import defaultdict -from earthkit.data.readers.grib.index import FieldListInFilesWithSqlIndex from earthkit.data.sources.directory import DirectorySource + +from earthkit.data.readers.grib.index import FieldListInFilesWithSqlIndex from earthkit.data.utils.progbar import progress_bar LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/sources/wekeo.py b/src/earthkit/data/sources/wekeo.py index 2675f2ef5..7876f8217 100644 --- a/src/earthkit/data/sources/wekeo.py +++ b/src/earthkit/data/sources/wekeo.py @@ -17,8 +17,7 @@ import yaml -from earthkit.data.utils.request import FileRequestRetriever -from earthkit.data.utils.request import RequestBuilder +from earthkit.data.utils.request import FileRequestRetriever, RequestBuilder from .file import FileSource from .prompt import APIKeyPrompt diff --git a/src/earthkit/data/sources/wekeocds.py b/src/earthkit/data/sources/wekeocds.py index cd03fa748..d730a7e22 100644 --- a/src/earthkit/data/sources/wekeocds.py +++ b/src/earthkit/data/sources/wekeocds.py @@ -15,13 +15,11 @@ raise ImportError("WEkEO access requires 'hda' to be installed") from earthkit.data.decorators import normalize -from earthkit.data.utils.request import FileRequestRetriever -from earthkit.data.utils.request import RequestBuilder +from earthkit.data.utils.request import FileRequestRetriever, RequestBuilder from .file import FileSource -from .wekeo import EXTENSIONS +from .wekeo import EXTENSIONS, HDAAPIKeyPrompt from .wekeo import ApiClient as WekeoClient -from .wekeo import HDAAPIKeyPrompt LOG = logging.getLogger(__name__) @@ -34,12 +32,10 @@ def __int__(self, *args, **kwargs): def retrieve(self, name, request, target=None): rq = {"dataset_id": name} - rq.update( - { - _name: (_value if isinstance(_value, list) or _name in ("data_format", "download_format") else [_value]) - for _name, _value in request.items() - } - ) + rq.update({ + _name: (_value if isinstance(_value, list) or _name in ("data_format", "download_format") else [_value]) + for _name, _value in request.items() + }) if "area" in request: rq.update({"bbox": request["area"]}) diff --git a/src/earthkit/data/sources/zarr.py b/src/earthkit/data/sources/zarr.py index 4b12c421d..5f0a61c25 100644 --- a/src/earthkit/data/sources/zarr.py +++ b/src/earthkit/data/sources/zarr.py @@ -12,7 +12,6 @@ class ZarrSource(Source): - def __init__(self, path, **kwargs): super().__init__(**kwargs) self._reader = ZarrReader(self, path, **kwargs) diff --git a/src/earthkit/data/sphinxext/generate_config_env_rst.py b/src/earthkit/data/sphinxext/generate_config_env_rst.py index 3f52d74d1..9b5230b3c 100644 --- a/src/earthkit/data/sphinxext/generate_config_env_rst.py +++ b/src/earthkit/data/sphinxext/generate_config_env_rst.py @@ -11,8 +11,7 @@ import re -from earthkit.data.core.config import CONFIG_AND_HELP -from earthkit.data.core.config import env_var_name +from earthkit.data.core.config import CONFIG_AND_HELP, env_var_name def execute(*args): diff --git a/src/earthkit/data/targets/__init__.py b/src/earthkit/data/targets/__init__.py index ef850c33b..13a027a41 100644 --- a/src/earthkit/data/targets/__init__.py +++ b/src/earthkit/data/targets/__init__.py @@ -9,8 +9,7 @@ import logging import os -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from functools import lru_cache from importlib import import_module @@ -21,8 +20,8 @@ class Target(metaclass=ABCMeta): """ Represent a target. - Parameters: - ----------- + Parameters + ---------- encoder: str, Encoder, None The encoder to use to encode the data. Can be overridden in the the :obj:`write` method. When a string is passed, the encoder is looked up in the available encoders. When None, @@ -55,8 +54,8 @@ def write( """ Write data to the target using the given encoder. - Parameters: - ----------- + Parameters + ---------- data: obj, None The data object to write. If None, the encoder will use all the other arguments to generate the data to write. @@ -74,8 +73,8 @@ def write( **kwargs: dict Other keyword arguments passed to the encoder. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ pass @@ -88,8 +87,8 @@ def close(self): It must also call :obj:`_mark_closed`. The target will not be able to write anymore. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ pass @@ -100,8 +99,8 @@ def flush(self): Some targets may require to flush the data to the underlying storage. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ pass @@ -151,8 +150,8 @@ def write( def _write(self, data, **kwargs): """Write generic data to the target. - Parameters: - ----------- + Parameters + ---------- data: Data to write to the target. """ @@ -251,12 +250,11 @@ def to_target(target, *args, **kwargs): This is a top level function that writes data to a target. - Parameters: - ----------- + Parameters + ---------- target: str The target to write to. Must be a string. """ - # data = kwargs.pop("data", None) with create_target(target, *args, **kwargs) as t: diff --git a/src/earthkit/data/targets/fdb.py b/src/earthkit/data/targets/fdb.py index a4eae34f6..b894d3a95 100644 --- a/src/earthkit/data/targets/fdb.py +++ b/src/earthkit/data/targets/fdb.py @@ -37,8 +37,8 @@ def close(self): The target will not be able to write anymore. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ self.flush() @@ -46,8 +46,8 @@ def close(self): def flush(self): """Flush the fdb. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ self._raise_if_closed() diff --git a/src/earthkit/data/targets/file.py b/src/earthkit/data/targets/file.py index 2e2a5833c..c38a2d361 100644 --- a/src/earthkit/data/targets/file.py +++ b/src/earthkit/data/targets/file.py @@ -20,8 +20,8 @@ class FileTarget(SimpleTarget): """ File target. - Parameters: - ----------- + Parameters + ---------- file: str, file-like, None The file path or file-like object to write to. When None, tries to guess the file name from the ``data`` if it is passed as a kwarg. @@ -34,8 +34,8 @@ class FileTarget(SimpleTarget): **kwargs: Additional keyword arguments passed to the parent class. - Raises: - ------- + Raises + ------ ValueError: If the file name is not specified and cannot be constructed. """ @@ -67,8 +67,8 @@ def close(self): If :obj:`FileTarget` was created with a file object this call has no effect. The target will not be able to write anymore. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ self._mark_closed() @@ -78,8 +78,8 @@ def close(self): def flush(self): """Flush the file. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ self._f().flush() @@ -96,14 +96,14 @@ def _f(self): return self._tmp_fileobj def _guess_filename(self, data=None): - """Try to guess filename from data when not provided""" + """Try to guess filename from data when not provided.""" if data is not None: for attr in ["source_filename", "path"]: if hasattr(self, attr) and getattr(self, attr) is not None: return [os.path.basename(getattr(self, attr))] def _check_overwrite(self, data): - """Ensure we do not overwrite file that is being read""" + """Ensure we do not overwrite file that is being read.""" if ( data is not None and self.filename is not None diff --git a/src/earthkit/data/targets/file_pattern.py b/src/earthkit/data/targets/file_pattern.py index 72dfa61c6..7f5c920e0 100644 --- a/src/earthkit/data/targets/file_pattern.py +++ b/src/earthkit/data/targets/file_pattern.py @@ -20,8 +20,8 @@ class FilePatternTarget(SimpleTarget): """ File target with a pattern for the output file names. - Parameters: - ----------- + Parameters + ---------- path: str The file path to write to. The output file name defines a pattern containing metadata keys in the format of ``{key}``. Each data item (e.g. a field) will be written into a file @@ -63,8 +63,8 @@ def close(self): The target will not be able to write anymore. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ self._mark_closed() diff --git a/src/earthkit/data/targets/zarr.py b/src/earthkit/data/targets/zarr.py index 9bb4fdd6f..f5196fdee 100644 --- a/src/earthkit/data/targets/zarr.py +++ b/src/earthkit/data/targets/zarr.py @@ -27,8 +27,8 @@ def close(self): The target will not be able to write anymore. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ pass @@ -36,8 +36,8 @@ def close(self): def flush(self): """Flush the fdb. - Raises: - ------- + Raises + ------ ValueError: If the target is already closed. """ pass diff --git a/src/earthkit/data/testing.py b/src/earthkit/data/testing.py index ce74a1234..7d2885546 100644 --- a/src/earthkit/data/testing.py +++ b/src/earthkit/data/testing.py @@ -16,18 +16,17 @@ from unittest.mock import patch import numpy as np - -from earthkit.data import from_object -from earthkit.data import from_source -from earthkit.data.readers.text import TextReader -from earthkit.data.sources.empty import EmptySource -from earthkit.data.sources.mars import StandaloneMarsClient from earthkit.utils.array import array_namespace as eku_array_namespace from earthkit.utils.array import convert as convert_array # from earthkit.utils.testing import get_array_backend from earthkit.utils.array.testing import NAMESPACE_DEVICES +from earthkit.data import from_object, from_source +from earthkit.data.readers.text import TextReader +from earthkit.data.sources.empty import EmptySource +from earthkit.data.sources.mars import StandaloneMarsClient + LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/translators/__init__.py b/src/earthkit/data/translators/__init__.py index 46f063eea..874d63cd8 100644 --- a/src/earthkit/data/translators/__init__.py +++ b/src/earthkit/data/translators/__init__.py @@ -11,9 +11,7 @@ class Translator(Base): def __call__(self): - """ - Return unmodified data. - """ + """Return unmodified data.""" return self.data @@ -39,5 +37,5 @@ def get_translator(source, cls, *args, **kwargs): def transform(*args, **kwargs): - """Executing wrapper for the get_translator class method""" + """Executing wrapper for the get_translator class method.""" return get_translator(*args, **kwargs)() diff --git a/src/earthkit/data/translators/pandas.py b/src/earthkit/data/translators/pandas.py index 07d11a857..cd4a6ff53 100644 --- a/src/earthkit/data/translators/pandas.py +++ b/src/earthkit/data/translators/pandas.py @@ -10,13 +10,13 @@ class PandasSeriesTranslator(Translator): - """Translator class for pandas `Series`""" + """Translator class for pandas `Series`.""" def __init__(self, data, *args, **kwargs): self.data = data.to_pandas(*args, **kwargs) def __call__(self): - """Series requested, if DataFrame return the first column""" + """Series requested, if DataFrame return the first column.""" import pandas as pd if isinstance(self.data, pd.DataFrame): @@ -26,7 +26,7 @@ def __call__(self): class PandasDataFrameTranslator(PandasSeriesTranslator): - """Translator class for pandas `DataFrame`""" + """Translator class for pandas `DataFrame`.""" def __call__(self): """Return DataFrame, if Series convert to DataFrame.""" @@ -39,7 +39,7 @@ def __call__(self): class GeoPandasDataFrameTranslator(PandasSeriesTranslator): - """Translator class for geopandas `DataFrame`""" + """Translator class for geopandas `DataFrame`.""" def __call__(self): """Return GeoDataFrame, if normal pandas convert to geopandas.""" diff --git a/src/earthkit/data/utils/availability.py b/src/earthkit/data/utils/availability.py index 8a97430b1..44d19ce39 100644 --- a/src/earthkit/data/utils/availability.py +++ b/src/earthkit/data/utils/availability.py @@ -18,11 +18,9 @@ import yaml from earthkit.data.utils import load_json_or_yaml -from earthkit.data.utils.factorise import Tree -from earthkit.data.utils.factorise import factorise +from earthkit.data.utils.factorise import Tree, factorise -from .humanize import dict_to_human -from .humanize import list_to_human +from .humanize import dict_to_human, list_to_human def _tidy_dict(query): diff --git a/src/earthkit/data/utils/batch.py b/src/earthkit/data/utils/batch.py index 6ff22028c..34af316c8 100644 --- a/src/earthkit/data/utils/batch.py +++ b/src/earthkit/data/utils/batch.py @@ -8,8 +8,7 @@ # -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod class Iter(metaclass=ABCMeta): diff --git a/src/earthkit/data/utils/bbox.py b/src/earthkit/data/utils/bbox.py index 467904e6c..7a5a73670 100644 --- a/src/earthkit/data/utils/bbox.py +++ b/src/earthkit/data/utils/bbox.py @@ -79,12 +79,12 @@ def make_invalid(cls): @property def width(self): - """number: Returns the East-West size (degrees)""" + """number: Returns the East-West size (degrees).""" return self.east - self.west @property def height(self): - """number: Returns the North-South size (degrees)""" + """number: Returns the North-South size (degrees).""" return self.north - self.south @classmethod diff --git a/src/earthkit/data/utils/diag.py b/src/earthkit/data/utils/diag.py index 567ed3c71..3578175e1 100644 --- a/src/earthkit/data/utils/diag.py +++ b/src/earthkit/data/utils/diag.py @@ -87,8 +87,7 @@ def current(self): def peak(self): """Peak rss memory usage in MB.""" - from resource import RUSAGE_SELF - from resource import getrusage + from resource import RUSAGE_SELF, getrusage rss = getrusage(RUSAGE_SELF).ru_maxrss return self.scale_to_mbytes(rss) diff --git a/src/earthkit/data/utils/factorise.py b/src/earthkit/data/utils/factorise.py index 9b256fe4c..b943b8de9 100644 --- a/src/earthkit/data/utils/factorise.py +++ b/src/earthkit/data/utils/factorise.py @@ -53,7 +53,7 @@ def split(self, dates): return result def overlaps(self, other): - """Returns the union of two intervals if they overlap, else None""" + """Returns the union of two intervals if they overlap, else None.""" s1, e1 = self.start, self.end + self.one s2, e2 = other.start, other.end + self.one s = max(s1, s2) @@ -64,7 +64,7 @@ def overlaps(self, other): return None def intersects(self, other): - """Returns the intersection of two intervals if they overlap, else None""" + """Returns the intersection of two intervals if they overlap, else None.""" s1, e1 = self.start, self.end + self.one s2, e2 = other.start, other.end + self.one s = max(s1, s2) diff --git a/src/earthkit/data/utils/html.py b/src/earthkit/data/utils/html.py index c8992a2da..6bd25fbc5 100644 --- a/src/earthkit/data/utils/html.py +++ b/src/earthkit/data/utils/html.py @@ -66,9 +66,7 @@ def table_from_dict(vals, title=None): t += """ {rows} -
""".format( - rows=" ".join([tr(k, v) for k, v in vals.items()]) - ) +""".format(rows=" ".join([tr(k, v) for k, v in vals.items()])) style = css("table") return style + t @@ -102,18 +100,16 @@ def tab(items, details=None, selected=None): """.format( - pages=" ".join( - [ - tab_page( - item["title"], - item.get("tooltip", ""), - str(uuid.uuid4()), - item["text"], - selected in (None, "") or item["title"] == selected, - ) - for item in items - ] - ), + pages=" ".join([ + tab_page( + item["title"], + item.get("tooltip", ""), + str(uuid.uuid4()), + item["text"], + selected in (None, "") or item["title"] == selected, + ) + for item in items + ]), ) style = css("tab") diff --git a/src/earthkit/data/utils/humanize.py b/src/earthkit/data/utils/humanize.py index cf1dfc5d6..37c1a6759 100644 --- a/src/earthkit/data/utils/humanize.py +++ b/src/earthkit/data/utils/humanize.py @@ -13,7 +13,11 @@ def bytes(n): - """>>> bytes(4096) + """Convert a number of bytes to a human readable string. + + Examples + -------- + >>> bytes(4096) '4 KiB' >>> bytes(4000) '3.9 KiB' @@ -32,20 +36,6 @@ def bytes(n): return "%s%g%s" % (sign, int(n * 10 + 0.5) / 10.0, u[i]) -def base2(n): - """>>> base2(4096) - '4K' - >>> base2(4000) - '3.9K' - """ - u = ["", "K", "M", "G", "T", " P", "E", "Z", "Y"] - i = 0 - while n >= 1024: - n /= 1024.0 - i += 1 - return "%g%s" % (int(n * 10 + 0.5) / 10.0, u[i]) - - PERIODS = ( (7 * 24 * 60 * 60, "week"), (24 * 60 * 60, "day"), diff --git a/src/earthkit/data/utils/message.py b/src/earthkit/data/utils/message.py index a711074c5..bd500d158 100644 --- a/src/earthkit/data/utils/message.py +++ b/src/earthkit/data/utils/message.py @@ -18,8 +18,7 @@ import eccodes import numpy as np -from earthkit.data.core.caching import CACHE -from earthkit.data.core.caching import auxiliary_cache_file +from earthkit.data.core.caching import CACHE, auxiliary_cache_file LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/utils/metadata/args.py b/src/earthkit/data/utils/metadata/args.py index 060098d9e..1089acdf1 100644 --- a/src/earthkit/data/utils/metadata/args.py +++ b/src/earthkit/data/utils/metadata/args.py @@ -9,7 +9,7 @@ def metadata_argument(*args, namespace=None, astype=None): - """Helps parsing the input arguments for the metadata methods""" + """Helps parsing the input arguments for the metadata methods.""" key = [] key_arg_type = None if len(args) == 1 and isinstance(args[0], str): diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index 1e5d5a03e..fc296d43a 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -14,13 +14,10 @@ import numpy as np from earthkit.data.core.geography import Geography -from earthkit.data.core.metadata import Metadata -from earthkit.data.core.metadata import MetadataAccessor +from earthkit.data.core.metadata import Metadata, MetadataAccessor from earthkit.data.decorators import thread_safe_cached_property from earthkit.data.utils.bbox import BoundingBox -from earthkit.data.utils.dates import datetime_from_grib -from earthkit.data.utils.dates import to_datetime -from earthkit.data.utils.dates import to_timedelta +from earthkit.data.utils.dates import datetime_from_grib, to_datetime, to_timedelta from earthkit.data.utils.projections import Projection LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/utils/meteo.py b/src/earthkit/data/utils/meteo.py index 40a5ee2c3..f8bc19470 100644 --- a/src/earthkit/data/utils/meteo.py +++ b/src/earthkit/data/utils/meteo.py @@ -15,7 +15,6 @@ import datetime import numpy as np - from earthkit.utils.array import array_namespace DAYS_PER_YEAR = 365.25 @@ -131,22 +130,18 @@ def _integrate( elif integration_order == 4: # slower, more accurate (4 points) _C1 = xp.sqrt(xp.asarray(6.0 / 5.0)) _C2 = xp.sqrt(xp.asarray(30)) - E = xp.asarray( - [ - -xp.sqrt(3.0 / 7.0 + 2.0 / 7.0 * _C1), - -xp.sqrt(3.0 / 7.0 - 2.0 / 7.0 * _C1), - xp.sqrt(3.0 / 7.0 - 2.0 / 7.0 * _C1), - xp.sqrt(3.0 / 7.0 + 2.0 / 7.0 * _C1), - ] - ) - W = xp.asarray( - [ - (18 - _C2) / 36, - (18 + _C2) / 36, - (18 + _C2) / 36, - (18 - _C2) / 36, - ] - ) + E = xp.asarray([ + -xp.sqrt(3.0 / 7.0 + 2.0 / 7.0 * _C1), + -xp.sqrt(3.0 / 7.0 - 2.0 / 7.0 * _C1), + xp.sqrt(3.0 / 7.0 - 2.0 / 7.0 * _C1), + xp.sqrt(3.0 / 7.0 + 2.0 / 7.0 * _C1), + ]) + W = xp.asarray([ + (18 - _C2) / 36, + (18 + _C2) / 36, + (18 + _C2) / 36, + (18 - _C2) / 36, + ]) else: raise ValueError("Invalid integration order %d", integration_order) diff --git a/src/earthkit/data/utils/module_inputs_wrapper.py b/src/earthkit/data/utils/module_inputs_wrapper.py index b934425ec..21a3cb3ce 100644 --- a/src/earthkit/data/utils/module_inputs_wrapper.py +++ b/src/earthkit/data/utils/module_inputs_wrapper.py @@ -1,3 +1,6 @@ +"""Module containing methods to transform the inputs of functions based on the function type setting, +common signatures or mapping defined at call time. +""" # (C) Copyright 2020 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -7,9 +10,6 @@ # nor does it submit to any jurisdiction. # -"""Module containing methods to transform the inputs of functions based on the function type setting, -common signitures or mapping defined at call time -""" import inspect import types import typing as T @@ -30,14 +30,14 @@ def _ensure_iterable(input_item): - """Ensure that an item is iterable""" + """Ensure that an item is iterable.""" if not isinstance(input_item, (tuple, list, dict)): return [input_item] return input_item def _ensure_tuple(input_item): - """Ensure that an item is a tuple""" + """Ensure that an item is a tuple.""" if not isinstance(input_item, tuple): return tuple(_ensure_iterable(input_item)) return input_item @@ -132,7 +132,7 @@ def signature_mapping(signature, kwarg_types): """Map args and kwargs to object types, using hierarchical selection method: 1. Explicitly defined type 2. Based on Type setting in function - 3. Do nothing + 3. Do nothing. """ mapping = {} for key, parameter in signature.parameters.items(): diff --git a/src/earthkit/data/utils/parts.py b/src/earthkit/data/utils/parts.py index 0130bb262..741c2710b 100644 --- a/src/earthkit/data/utils/parts.py +++ b/src/earthkit/data/utils/parts.py @@ -7,8 +7,7 @@ # nor does it submit to any jurisdiction. import os -from collections import defaultdict -from collections import namedtuple +from collections import defaultdict, namedtuple SimplePart = namedtuple("SimplePart", ["offset", "length"]) @@ -54,7 +53,7 @@ def __repr__(self): def check_urls_and_parts(urls, parts): - """Check if urls and parts are compatible + """Check if urls and parts are compatible. When any of the ``urls`` contain a part ``parts`` must be None. """ @@ -148,7 +147,8 @@ class PathAndParts: def __init__(self, path, parts): """Should not be instantiated directly. - The public API are the factory methods.""" + The public API are the factory methods. + """ self.path = path self.parts = parts diff --git a/src/earthkit/data/utils/patterns.py b/src/earthkit/data/utils/patterns.py index 9e6614049..e82e818fd 100644 --- a/src/earthkit/data/utils/patterns.py +++ b/src/earthkit/data/utils/patterns.py @@ -13,16 +13,11 @@ import re from pathlib import Path from typing import Any as TypingAny -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union +from typing import Dict, List, Optional, Tuple, Union from earthkit.data.decorators import thread_safe_cached_property -from .dates import to_datetime -from .dates import to_timedelta +from .dates import to_datetime, to_timedelta LOG = logging.getLogger(__name__) @@ -406,6 +401,7 @@ def substitute_many(self, params: Dict[str, TypingAny]) -> Optional[List[TypingA ------- list List of substituted values. + Raises ------ ValueError diff --git a/src/earthkit/data/utils/projections/__init__.py b/src/earthkit/data/utils/projections/__init__.py index 4e5d768a5..c6deb3c13 100644 --- a/src/earthkit/data/utils/projections/__init__.py +++ b/src/earthkit/data/utils/projections/__init__.py @@ -6,8 +6,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from . import cf -from . import proj +from . import cf, proj try: import cartopy.crs as ccrs @@ -20,7 +19,7 @@ DEFAULT_LATLON_PROJ_STRING = ( - "+proj=eqc +ellps=WGS84 +a=6378137.0 +lon_0=0.0 +to_meter=111319.4907932736 " "+no_defs +type=crs" + "+proj=eqc +ellps=WGS84 +a=6378137.0 +lon_0=0.0 +to_meter=111319.4907932736 +no_defs +type=crs" ) diff --git a/src/earthkit/data/utils/request.py b/src/earthkit/data/utils/request.py index 648c4400e..b86571e62 100644 --- a/src/earthkit/data/utils/request.py +++ b/src/earthkit/data/utils/request.py @@ -10,8 +10,7 @@ import itertools import logging import sys -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from earthkit.data.core.thread import SoftThreadPool from earthkit.data.decorators import thread_safe_cached_property @@ -42,8 +41,8 @@ def __init__(self, owner, *args, request=None, normaliser=None, **kwargs): owner : Any The owner of the request builder. *args : tuple - Positional arguments representing request dictionaries. Each item can be dictionary - or a list/tuple of dictionaries. + Positional arguments representing request dictionaries. + Each item can be dictionary or a list/tuple of dictionaries. **kwargs : dict Keyword arguments representing request parameters. request : dict or list/tuple of dict, optional diff --git a/src/earthkit/data/utils/summary.py b/src/earthkit/data/utils/summary.py index 80c41875e..cc3bce1da 100644 --- a/src/earthkit/data/utils/summary.py +++ b/src/earthkit/data/utils/summary.py @@ -137,8 +137,7 @@ def _repr_html_(self): from earthkit.data.core.ipython import ipython_active if ipython_active: - from earthkit.data.utils.html import tab - from earthkit.data.utils.html import table_from_dict + from earthkit.data.utils.html import tab, table_from_dict if len(self.data) == 1: return table_from_dict(self.data[0]["data"]) @@ -224,13 +223,11 @@ def _parse_dump(self, data, parent): vals = v["value"] if isinstance(vals, list): vals = self._format_list(vals) - parent.append( - { - "key": v["key"], - "value": vals, - "units": v.get("units", None), - } - ) + parent.append({ + "key": v["key"], + "value": vals, + "units": v.get("units", None), + }) else: if arrayCnt > 1: item = [] diff --git a/src/earthkit/data/utils/url.py b/src/earthkit/data/utils/url.py index 2ab32a159..49ce44f4e 100644 --- a/src/earthkit/data/utils/url.py +++ b/src/earthkit/data/utils/url.py @@ -55,7 +55,8 @@ class UrlSourcePathAndParts(PathAndParts): class UrlSpec: def __init__(self, spec, urls_and_parts): """Should not be instantiated directly. - The public API are the factory methods.""" + The public API are the factory methods. + """ self.spec = spec self.url_and_parts = urls_and_parts diff --git a/src/earthkit/data/utils/xarray/attrs.py b/src/earthkit/data/utils/xarray/attrs.py index 724f90b2f..1714a0a71 100644 --- a/src/earthkit/data/utils/xarray/attrs.py +++ b/src/earthkit/data/utils/xarray/attrs.py @@ -9,13 +9,11 @@ import logging import os -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from collections import defaultdict from earthkit.data.decorators import thread_safe_cached_property -from earthkit.data.utils import ensure_dict -from earthkit.data.utils import ensure_iterable +from earthkit.data.utils import ensure_dict, ensure_iterable LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/utils/xarray/builder.py b/src/earthkit/data/utils/xarray/builder.py index 4c9feda09..cb4f7e91f 100644 --- a/src/earthkit/data/utils/xarray/builder.py +++ b/src/earthkit/data/utils/xarray/builder.py @@ -8,14 +8,12 @@ # import logging -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod import xarray import xarray.core.indexing as indexing -from earthkit.data.utils import ensure_dict -from earthkit.data.utils import ensure_iterable +from earthkit.data.utils import ensure_dict, ensure_iterable from .dim import LevelPerTypeDim from .profile import Profile @@ -390,7 +388,7 @@ def build_values(self, *args, **kwargs): pass def pre_build_variables(self): - """Generate a builder for each variable""" + """Generate a builder for each variable.""" builders = {} if self.profile.variable.is_mono: @@ -525,7 +523,7 @@ class TensorBackendDataBuilder(BackendDataBuilder): """ def build_values(self, tensor, var_dims, name): - """Generate the data object stored in the xarray variable""" + """Generate the data object stored in the xarray variable.""" # There is no need for the extra structures in the wrapped source in the # tensor any longer. It is replaced by the original unwrapped fieldlist. tensor.source = tensor.source.unwrap() @@ -552,8 +550,7 @@ class MemoryBackendDataBuilder(BackendDataBuilder): """ def build_values(self, tensor, var_dims, name): - """Generate the data object stored in the xarray variable""" - + """Generate the data object stored in the xarray variable.""" # At this point all the fields must be a ReleasableField. # We mark the fields so that their data will be released on the next # values access. diff --git a/src/earthkit/data/utils/xarray/cf_attrs.yaml b/src/earthkit/data/utils/xarray/cf_attrs.yaml index f83962ee7..03767eb3b 100644 --- a/src/earthkit/data/utils/xarray/cf_attrs.yaml +++ b/src/earthkit/data/utils/xarray/cf_attrs.yaml @@ -8,18 +8,18 @@ Conventions: _FillValue: type: D use: - - C - - D + - C + - D actual_range: type: N use: - - C - - D + - C + - D add_offset: type: N use: - - C - - D + - C + - D ancillary_variables: type: S use: D @@ -47,9 +47,9 @@ climatology: comment: type: S use: - - G - - C - - D + - G + - C + - D compress: type: S use: C @@ -59,8 +59,8 @@ computed_standard_name: coordinates: type: S use: - - D - - M + - D + - M external_variables: type: S use: G @@ -82,29 +82,29 @@ formula_terms: geometry: type: S use: - - C - - D + - C + - D geometry_type: type: S use: M grid_mapping: type: S use: - - D - - M + - D + - M history: type: S use: - - G - - Gr + - G + - Gr instance_dimension: type: S use: "-" institution: type: S use: - - G - - D + - G + - D interior_ring: type: S use: M @@ -117,13 +117,13 @@ leap_year: long_name: type: S use: - - C - - D + - C + - D missing_value: type: D use: - - C - - D + - C + - D month_lengths: type: N use: C @@ -145,51 +145,51 @@ positive: references: type: S use: - - G - - D + - G + - D sample_dimension: type: S use: "-" scale_factor: type: N use: - - C - - D + - C + - D source: type: S use: - - G - - D + - G + - D standard_error_multiplier: type: N use: D standard_name: type: S use: - - C - - D + - C + - D title: type: S use: - - G - - Gr + - G + - Gr units: type: S use: - - C - - D + - C + - D valid_max: type: N use: - - C - - D + - C + - D valid_min: type: N use: - - C - - D + - C + - D valid_range: type: N use: - - C - - D + - C + - D diff --git a/src/earthkit/data/utils/xarray/check.py b/src/earthkit/data/utils/xarray/check.py index 6f2998fc4..1b208fd80 100644 --- a/src/earthkit/data/utils/xarray/check.py +++ b/src/earthkit/data/utils/xarray/check.py @@ -10,8 +10,7 @@ import logging import math -from .diff import DictDiff -from .diff import ListDiff +from .diff import DictDiff, ListDiff LOG = logging.getLogger(__name__) diff --git a/src/earthkit/data/utils/xarray/coord.py b/src/earthkit/data/utils/xarray/coord.py index 8450b1e7a..71bfe0e57 100644 --- a/src/earthkit/data/utils/xarray/coord.py +++ b/src/earthkit/data/utils/xarray/coord.py @@ -12,12 +12,7 @@ import numpy as np -from .dim import DATE_KEYS -from .dim import DATETIME_KEYS -from .dim import LEVEL_KEYS -from .dim import MONTH_KEYS -from .dim import STEP_KEYS -from .dim import TIME_KEYS +from .dim import DATE_KEYS, DATETIME_KEYS, LEVEL_KEYS, MONTH_KEYS, STEP_KEYS, TIME_KEYS LOG = logging.getLogger(__name__) @@ -124,8 +119,7 @@ def convert(self, profile): class TimeCoord(Coord): def convert(self, profile): if profile.decode_timedelta: - from earthkit.data.utils.dates import to_time - from earthkit.data.utils.dates import to_timedelta + from earthkit.data.utils.dates import to_time, to_timedelta return [to_timedelta(to_time(x)) for x in self.vals] return super().convert(profile) diff --git a/src/earthkit/data/utils/xarray/coords.yaml b/src/earthkit/data/utils/xarray/coords.yaml index 41137f5ab..3020a484a 100644 --- a/src/earthkit/data/utils/xarray/coords.yaml +++ b/src/earthkit/data/utils/xarray/coords.yaml @@ -22,28 +22,28 @@ coord_attrs: standard_name: time long_name: valid_time level_coord_attrs: - - key: typeOfLevel - isobaricInhPa: - units: hPa - positive: down - stored_direction: decreasing - standard_name: air_pressure - long_name: pressure - isobaricInPa: - units: Pa - positive: down - stored_direction: decreasing - standard_name: air_pressure - long_name: pressure - heightAboveGround: - units: m - positive: up - long_name: height above the surface - standard_name: height - - key: levtype - pl: - units: hPa - positive: down - stored_direction: decreasing - standard_name: air_pressure - long_name: pressure +- key: typeOfLevel + isobaricInhPa: + units: hPa + positive: down + stored_direction: decreasing + standard_name: air_pressure + long_name: pressure + isobaricInPa: + units: Pa + positive: down + stored_direction: decreasing + standard_name: air_pressure + long_name: pressure + heightAboveGround: + units: m + positive: up + long_name: height above the surface + standard_name: height +- key: levtype + pl: + units: hPa + positive: down + stored_direction: decreasing + standard_name: air_pressure + long_name: pressure diff --git a/src/earthkit/data/utils/xarray/dim.py b/src/earthkit/data/utils/xarray/dim.py index a9257a2d8..7976a879d 100644 --- a/src/earthkit/data/utils/xarray/dim.py +++ b/src/earthkit/data/utils/xarray/dim.py @@ -9,8 +9,7 @@ import logging -from earthkit.data.utils import ensure_dict -from earthkit.data.utils import ensure_iterable +from earthkit.data.utils import ensure_dict, ensure_iterable from earthkit.data.utils.dates import datetime_from_grib LOG = logging.getLogger(__name__) @@ -195,7 +194,7 @@ def update(self, ds): # sanity check if self.profile.variable.key in self: - raise ValueError((f"Variable key {self.profile.variable.key} cannot be in " f"dimension={self.name}")) + raise ValueError((f"Variable key {self.profile.variable.key} cannot be in dimension={self.name}")) # print(f"key={self.key} index={ds.index(self.key)}") @@ -916,22 +915,20 @@ def rename_dataset_dims(self, dataset): PREDEFINED_DIMS = {} -for i, d in enumerate( - [ - NumberDim, - ForecastRefTimeDim, - DateDim, - TimeDim, - StepDim, - ValidTimeDim, - IndexingTimeDim, - ReferenceTimeDim, - LevelDim, - LevelTypeDim, - LevelPerTypeDim, - LevelAndTypeDim, - ] -): +for i, d in enumerate([ + NumberDim, + ForecastRefTimeDim, + DateDim, + TimeDim, + StepDim, + ValidTimeDim, + IndexingTimeDim, + ReferenceTimeDim, + LevelDim, + LevelTypeDim, + LevelPerTypeDim, + LevelAndTypeDim, +]): if d.name: PREDEFINED_DIMS[d.name] = d else: diff --git a/src/earthkit/data/utils/xarray/engine.py b/src/earthkit/data/utils/xarray/engine.py index 650969d9a..b008b8383 100644 --- a/src/earthkit/data/utils/xarray/engine.py +++ b/src/earthkit/data/utils/xarray/engine.py @@ -458,8 +458,7 @@ def to_grib(self, filename): import warnings warnings.warn( - "The `to_grib` is deprecated in 0.15.0 and will be removed in a future version. " - "Use `to_target` instead.", + "The `to_grib` is deprecated in 0.15.0 and will be removed in a future version. Use `to_target` instead.", DeprecationWarning, ) from earthkit.data.targets import create_target @@ -498,8 +497,7 @@ def metadata(self): if "message" in md: data = md["message"] from earthkit.data.readers.grib.memory import GribMessageMemoryReader - from earthkit.data.readers.grib.metadata import StandAloneGribMetadata - from earthkit.data.readers.grib.metadata import WrappedMetadata + from earthkit.data.readers.grib.metadata import StandAloneGribMetadata, WrappedMetadata handle = next(GribMessageMemoryReader(data)).handle bpv = md.get("bitsPerValue", 0) diff --git a/src/earthkit/data/utils/xarray/fieldlist.py b/src/earthkit/data/utils/xarray/fieldlist.py index cffcb9443..ce37ffe32 100644 --- a/src/earthkit/data/utils/xarray/fieldlist.py +++ b/src/earthkit/data/utils/xarray/fieldlist.py @@ -12,8 +12,7 @@ from collections import defaultdict from earthkit.data.core.fieldlist import FieldList -from earthkit.data.core.index import Selection -from earthkit.data.core.index import normalize_selection +from earthkit.data.core.index import Selection, normalize_selection from earthkit.data.core.order import build_remapping LOG = logging.getLogger(__name__) @@ -223,9 +222,9 @@ def unique_values(self, names, component=False): if k in self.remapping: indices[k] = [x[0] for x in v] components[k] = self.remapping.components(k), [x[1] for x in v] - assert len(indices[k]) == len( - components[k][1] - ), f"{len(indices[k])} != {len(components[k])} {indices[k]=} {components[k]=}" + assert len(indices[k]) == len(components[k][1]), ( + f"{len(indices[k])} != {len(components[k])} {indices[k]=} {components[k]=}" + ) else: indices[k] = v else: diff --git a/src/earthkit/data/utils/xarray/grib.py b/src/earthkit/data/utils/xarray/grib.py index f4d52eefd..292e06e7f 100644 --- a/src/earthkit/data/utils/xarray/grib.py +++ b/src/earthkit/data/utils/xarray/grib.py @@ -11,12 +11,14 @@ import logging from itertools import product -from earthkit.data.utils.dates import date_to_grib -from earthkit.data.utils.dates import datetime_to_grib -from earthkit.data.utils.dates import step_to_grib -from earthkit.data.utils.dates import time_to_grib -from earthkit.data.utils.dates import to_datetime -from earthkit.data.utils.dates import to_timedelta +from earthkit.data.utils.dates import ( + date_to_grib, + datetime_to_grib, + step_to_grib, + time_to_grib, + to_datetime, + to_timedelta, +) LOG = logging.getLogger(__name__) @@ -143,7 +145,6 @@ def data_array_to_fields(da, metadata=None): step_len = None for values in product(*[coords[dim] for dim in dims]): - # field local_coords = dict(zip(dims, values)) for k in components: diff --git a/src/earthkit/data/utils/xarray/grib.yaml b/src/earthkit/data/utils/xarray/grib.yaml index 2037fd1bd..33801cc5b 100644 --- a/src/earthkit/data/utils/xarray/grib.yaml +++ b/src/earthkit/data/utils/xarray/grib.yaml @@ -8,10 +8,10 @@ dim_roles: variable_key: param attrs_mode: fixed variable_attrs: - - cfName - - name - - units - - typeOfLevel +- cfName +- name +- units +- typeOfLevel global_attrs: - - Conventions: CF-1.8 - - institution: ECMWF +- Conventions: CF-1.8 +- institution: ECMWF diff --git a/src/earthkit/data/utils/xarray/mars.yaml b/src/earthkit/data/utils/xarray/mars.yaml index 0f17e91a4..1f913ecd3 100644 --- a/src/earthkit/data/utils/xarray/mars.yaml +++ b/src/earthkit/data/utils/xarray/mars.yaml @@ -8,37 +8,37 @@ dim_roles: variable_key: param attrs_mode: unique attrs: - - class - - stream - - levtype - - type - - expver - - date - - hdate - - andate - - time - - antime - - reference - - anoffset - - verify - - fcmonth - - fcperiod - - leadtime - - opttime - - origin - - domain - - method - - diagnostic - - iteration - - number - - quantile - - levelist +- class +- stream +- levtype +- type +- expver +- date +- hdate +- andate +- time +- antime +- reference +- anoffset +- verify +- fcmonth +- fcperiod +- leadtime +- opttime +- origin +- domain +- method +- diagnostic +- iteration +- number +- quantile +- levelist variable_attrs: - - param - - cfName - - name - - paramId - - units +- param +- cfName +- name +- paramId +- units global_attrs: - - Conventions: CF-1.8 - - institution: ECMWF +- Conventions: CF-1.8 +- institution: ECMWF diff --git a/src/earthkit/data/utils/xarray/profile.py b/src/earthkit/data/utils/xarray/profile.py index 77ae03272..0760abb88 100644 --- a/src/earthkit/data/utils/xarray/profile.py +++ b/src/earthkit/data/utils/xarray/profile.py @@ -10,12 +10,10 @@ import logging import os import threading -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from earthkit.data.decorators import thread_safe_cached_property -from earthkit.data.utils import ensure_dict -from earthkit.data.utils import ensure_iterable +from earthkit.data.utils import ensure_dict, ensure_iterable LOG = logging.getLogger(__name__) @@ -140,7 +138,6 @@ def update(self, ds): ds: fieldlist FieldList object with cached metadata """ - self.variables = ds.index(self.key) if self.drop: @@ -342,7 +339,7 @@ def from_conf(cls, name, conf, *args, **kwargs): opt = copy.deepcopy(PROFILE_CONF.defaults) def _deprec_array_module(data): - """Deprecated: use 'array_namespace' instead""" + """Deprecated: use 'array_namespace' instead.""" if "array_module" in data: import warnings @@ -352,7 +349,7 @@ def _deprec_array_module(data): data["array_namespace"] = array_module def _deprec_array_backend(data): - """Deprecated: use 'array_namespace' instead""" + """Deprecated: use 'array_namespace' instead.""" if "array_backend" in data: import warnings @@ -402,7 +399,7 @@ def _check_type(k, v, t): @classmethod def to_docs(cls, name): - """Used to generate documentation""" + """Used to generate documentation.""" import copy if name is None: diff --git a/src/earthkit/data/utils/xarray/splitter.py b/src/earthkit/data/utils/xarray/splitter.py index 2bc9f8304..bc1a4787b 100644 --- a/src/earthkit/data/utils/xarray/splitter.py +++ b/src/earthkit/data/utils/xarray/splitter.py @@ -8,8 +8,7 @@ # import logging -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from itertools import product LOG = logging.getLogger(__name__) @@ -27,7 +26,7 @@ def split(self, ds, profile): @staticmethod def make(split_dims, auto_split=False): - """TODO: auto_split is not implemented""" + """TODO: auto_split is not implemented.""" if not auto_split and not split_dims: return NoSplitter() elif split_dims: diff --git a/src/earthkit/data/wrappers/pandas.py b/src/earthkit/data/wrappers/pandas.py index f90e0c988..3022c6aee 100644 --- a/src/earthkit/data/wrappers/pandas.py +++ b/src/earthkit/data/wrappers/pandas.py @@ -65,7 +65,7 @@ def to_xarray(self, **kwargs): def to_netcdf(self, *args, **kwargs): """ - Write the pandas object to a netCDF file + Write the pandas object to a netCDF file. Returns ------- @@ -75,7 +75,7 @@ def to_netcdf(self, *args, **kwargs): def to_json(self, *args, **kwargs): """ - Write the pandas object to a netCDF file + Write the pandas object to a netCDF file. Returns ------- @@ -86,7 +86,7 @@ def to_json(self, *args, **kwargs): def describe(self): """ - A pandas is tabular, therefore return itself + A pandas is tabular, therefore return itself. Returns ------- @@ -146,7 +146,7 @@ def __init__(self, source): self.fields = None def __iter__(self): - """Iterate over features in geojson via pandas""" + """Iterate over features in geojson via pandas.""" self._scan() return iter(self.fields) @@ -162,11 +162,11 @@ def _scan(self): self.fields = self.get_fields() def get_fields(self): - """For geopandas, a field is a feature""" + """For geopandas, a field is a feature.""" return [row[1] for row in self.data.iterrows()] def bounding_box(self): - """For geopandas, get bounding box and convert to EK.BoundingBox type""" + """For geopandas, get bounding box and convert to EK.BoundingBox type.""" return BoundingBox(north=self.north, south=self.south, east=self.east, west=self.west) diff --git a/src/earthkit/data/wrappers/string.py b/src/earthkit/data/wrappers/string.py index 84a9deab1..67bea073d 100644 --- a/src/earthkit/data/wrappers/string.py +++ b/src/earthkit/data/wrappers/string.py @@ -9,8 +9,7 @@ import datetime import re -from dateutil.parser import isoparse -from dateutil.parser import parse +from dateutil.parser import isoparse, parse from earthkit.data.wrappers import Wrapper diff --git a/tests/ai-models/test_ai_warpped_field.py b/tests/ai-models/test_ai_warpped_field.py index 3c0a9a55d..4ef7568b4 100644 --- a/tests/ai-models/test_ai_warpped_field.py +++ b/tests/ai-models/test_ai_warpped_field.py @@ -13,8 +13,7 @@ import numpy as np -from earthkit.data import SimpleFieldList -from earthkit.data import from_source +from earthkit.data import SimpleFieldList, from_source from earthkit.data.testing import earthkit_examples_file LOG = logging.getLogger(__name__) diff --git a/tests/array_fieldlist/array_fl_fixtures.py b/tests/array_fieldlist/array_fl_fixtures.py index 8ff3aa44c..e5869c6a8 100644 --- a/tests/array_fieldlist/array_fl_fixtures.py +++ b/tests/array_fieldlist/array_fl_fixtures.py @@ -11,11 +11,12 @@ import os +from earthkit.utils.array import array_namespace as eku_array_namespace + from earthkit.data import from_source from earthkit.data.core.fieldlist import FieldList from earthkit.data.core.temporary import temp_file from earthkit.data.testing import earthkit_examples_file -from earthkit.utils.array import array_namespace as eku_array_namespace def load_array_fl(num, array_namespace=None): diff --git a/tests/array_fieldlist/conftest.py b/tests/array_fieldlist/conftest.py new file mode 100644 index 000000000..e669d95eb --- /dev/null +++ b/tests/array_fieldlist/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py index 2ac6146f8..c0d59414b 100644 --- a/tests/array_fieldlist/test_array_field_usermetadata.py +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -20,14 +20,12 @@ def test_array_field_usermetadata_nogeom(): vals = np.linspace(0, 1, 10) - meta = UserMetadata( - { - "shortName": "test", - "longName": "Test", - "date": 20180801, - "time": 300, - } - ) + meta = UserMetadata({ + "shortName": "test", + "longName": "Test", + "date": 20180801, + "time": 300, + }) f = ArrayField(vals, meta) diff --git a/tests/array_fieldlist/test_numpy_fl_write.py b/tests/array_fieldlist/test_numpy_fl_write.py index 3ae0e76dd..c777fd82c 100644 --- a/tests/array_fieldlist/test_numpy_fl_write.py +++ b/tests/array_fieldlist/test_numpy_fl_write.py @@ -11,24 +11,22 @@ import logging import os -import sys import numpy as np import pytest +from array_fl_fixtures import load_array_fl from earthkit.data import from_source from earthkit.data.core.fieldlist import FieldList from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import ARRAY_BACKENDS -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import check_array_type -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file -from earthkit.data.testing import write_to_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from array_fl_fixtures import load_array_fl # noqa: E402 +from earthkit.data.testing import ( + ARRAY_BACKENDS, + WRITE_TO_FILE_METHODS, + check_array_type, + earthkit_examples_file, + earthkit_test_data_file, + write_to_file, +) LOG = logging.getLogger(__name__) diff --git a/tests/array_fieldlist/test_numpy_fs.py b/tests/array_fieldlist/test_numpy_fs.py index b15179653..38726682a 100644 --- a/tests/array_fieldlist/test_numpy_fs.py +++ b/tests/array_fieldlist/test_numpy_fs.py @@ -10,22 +10,18 @@ # import os -import sys import numpy as np import pytest +from array_fl_fixtures import ( + check_array_fl, + check_array_fl_from_to_fieldlist, +) from earthkit.data import from_source from earthkit.data.core.fieldlist import FieldList from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import write_to_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from array_fl_fixtures import check_array_fl # noqa: E402 -from array_fl_fixtures import check_array_fl_from_to_fieldlist # noqa: E402 +from earthkit.data.testing import WRITE_TO_FILE_METHODS, earthkit_examples_file, write_to_file @pytest.mark.parametrize("write_method", WRITE_TO_FILE_METHODS) diff --git a/tests/array_fieldlist/test_numpy_fs_concat.py b/tests/array_fieldlist/test_numpy_fs_concat.py index 4969183c4..ebe3e75ff 100644 --- a/tests/array_fieldlist/test_numpy_fs_concat.py +++ b/tests/array_fieldlist/test_numpy_fs_concat.py @@ -9,20 +9,17 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from array_fl_fixtures import ( + check_array_fl, + check_save_to_disk, + load_array_fl, +) from earthkit.data import from_source from earthkit.data.core.fieldlist import FieldList -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from array_fl_fixtures import check_array_fl # noqa: E402 -from array_fl_fixtures import check_save_to_disk # noqa: E402 -from array_fl_fixtures import load_array_fl # noqa: E402 - @pytest.mark.parametrize("mode", ["oper", "multi"]) def test_array_fl_grib_concat_2a(mode): diff --git a/tests/array_fieldlist/test_numpy_fs_metadata.py b/tests/array_fieldlist/test_numpy_fs_metadata.py index a437db726..efb6e352f 100644 --- a/tests/array_fieldlist/test_numpy_fs_metadata.py +++ b/tests/array_fieldlist/test_numpy_fs_metadata.py @@ -9,15 +9,12 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from array_fl_fixtures import load_array_fl # noqa: E402 -from array_fl_fixtures import load_array_fl_file # noqa: E402 +from array_fl_fixtures import ( + load_array_fl, + load_array_fl_file, +) # Note: Almost all grib metadata tests are also run for numpyfs. # See grib/test_grib_metadata.py diff --git a/tests/array_fieldlist/test_numpy_fs_summary.py b/tests/array_fieldlist/test_numpy_fs_summary.py index a0c7d5eb9..f4b46f2bd 100644 --- a/tests/array_fieldlist/test_numpy_fs_summary.py +++ b/tests/array_fieldlist/test_numpy_fs_summary.py @@ -9,12 +9,8 @@ # nor does it submit to any jurisdiction. # -import os -import sys -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from array_fl_fixtures import load_array_fl_file # noqa: E402 +from array_fl_fixtures import load_array_fl_file # Note: Almost all grib metadata tests are also run for numpyfs. # See grib/test_grib_summary.py diff --git a/tests/bufr/test_bufr_concat.py b/tests/bufr/test_bufr_concat.py index 6839b70bf..a7104e02c 100644 --- a/tests/bufr/test_bufr_concat.py +++ b/tests/bufr/test_bufr_concat.py @@ -10,8 +10,7 @@ # from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_examples_file def test_bufr_concat(): diff --git a/tests/bufr/test_bufr_contents.py b/tests/bufr/test_bufr_contents.py index b17047df2..21d6cd547 100644 --- a/tests/bufr/test_bufr_contents.py +++ b/tests/bufr/test_bufr_contents.py @@ -10,8 +10,7 @@ from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_test_data_file def test_bufr_iteration(): diff --git a/tests/bufr/test_bufr_convert.py b/tests/bufr/test_bufr_convert.py index 9c37a3dbf..015487811 100644 --- a/tests/bufr/test_bufr_convert.py +++ b/tests/bufr/test_bufr_convert.py @@ -44,13 +44,11 @@ def test_bufr_to_pandas_filters(): filters={"WMO_station_id": [2836, 2963]}, ) - ref = pd.DataFrame.from_dict( - { - "latitude": [67.37, 60.82], - "longitude": [26.63, 23.50], - "WMO_station_id": [2836, 2963], - } - ) + ref = pd.DataFrame.from_dict({ + "latitude": [67.37, 60.82], + "longitude": [26.63, 23.50], + "WMO_station_id": [2836, 2963], + }) assert_frame_equal(res, ref[res.columns]) diff --git a/tests/bufr/test_bufr_summary.py b/tests/bufr/test_bufr_summary.py index b9d7afa11..18f8dae96 100644 --- a/tests/bufr/test_bufr_summary.py +++ b/tests/bufr/test_bufr_summary.py @@ -11,8 +11,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_test_data_file def test_bufr_ls_invalid_num(): diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py index d884a8d57..aeb5139e9 100644 --- a/tests/core/test_cache.py +++ b/tests/core/test_cache.py @@ -13,13 +13,10 @@ import pytest -from earthkit.data import cache -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import cache, config, from_source from earthkit.data.core.caching import cache_file from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_examples_file def check_cache_files(dir_path, managed=True): @@ -109,12 +106,10 @@ def test_cache_policy(): # cache = temporary with user defined root path with temp_directory() as root_dir: - with config.temporary( - { - "cache-policy": "temporary", - "temporary-cache-directory-root": root_dir, - } - ): + with config.temporary({ + "cache-policy": "temporary", + "temporary-cache-directory-root": root_dir, + }): assert config.get("cache-policy") == "temporary" assert config.get("temporary-cache-directory-root") == root_dir assert cache.policy.managed() is True @@ -269,12 +264,10 @@ def test_cache_management(policy): config.set({"cache-policy": "user", "user-cache-directory": tmp_dir_path}) assert cache.directory() == tmp_dir_path elif policy == "temporary": - config.set( - { - "cache-policy": "temporary", - "temporary-cache-directory-root": tmp_dir_path, - } - ) + config.set({ + "cache-policy": "temporary", + "temporary-cache-directory-root": tmp_dir_path, + }) assert os.path.dirname(cache.directory()) == tmp_dir_path else: assert False diff --git a/tests/core/test_cache_with_settings.py b/tests/core/test_cache_with_settings.py index 98f9f23a4..f9edadabb 100644 --- a/tests/core/test_cache_with_settings.py +++ b/tests/core/test_cache_with_settings.py @@ -13,13 +13,10 @@ import pytest -from earthkit.data import cache -from earthkit.data import from_source -from earthkit.data import settings +from earthkit.data import cache, from_source, settings from earthkit.data.core.caching import cache_file from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_examples_file def check_cache_files(dir_path, managed=True): @@ -109,12 +106,10 @@ def test_cache_policy_setting(): # cache = temporary with user defined root path with temp_directory() as root_dir: - with settings.temporary( - { - "cache-policy": "temporary", - "temporary-cache-directory-root": root_dir, - } - ): + with settings.temporary({ + "cache-policy": "temporary", + "temporary-cache-directory-root": root_dir, + }): assert settings.get("cache-policy") == "temporary" assert settings.get("temporary-cache-directory-root") == root_dir assert cache.policy.managed() is True @@ -269,12 +264,10 @@ def test_cache_management_setting(policy): settings.set({"cache-policy": "user", "user-cache-directory": tmp_dir_path}) assert cache.directory() == tmp_dir_path elif policy == "temporary": - settings.set( - { - "cache-policy": "temporary", - "temporary-cache-directory-root": tmp_dir_path, - } - ) + settings.set({ + "cache-policy": "temporary", + "temporary-cache-directory-root": tmp_dir_path, + }) assert os.path.dirname(cache.directory()) == tmp_dir_path else: assert False diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 82fb1738e..26dfca3b5 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -14,8 +14,7 @@ import pytest from earthkit.data import config -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file +from earthkit.data.core.temporary import temp_directory, temp_file def read_config_yaml(path=os.path.expanduser("~/.config/earthkit/data/config.yaml")): diff --git a/tests/core/test_fieldlist.py b/tests/core/test_fieldlist.py index 84679b02d..f39c47a93 100644 --- a/tests/core/test_fieldlist.py +++ b/tests/core/test_fieldlist.py @@ -11,9 +11,9 @@ import numpy as np import pytest +from earthkit.utils.array.testing.testing import NO_TORCH from earthkit.data import SimpleFieldList -from earthkit.utils.array.testing.testing import NO_TORCH def test_empty_fieldlist_values_numpy(): diff --git a/tests/core/test_metadata.py b/tests/core/test_metadata.py index 1e115c4ba..afd40f307 100644 --- a/tests/core/test_metadata.py +++ b/tests/core/test_metadata.py @@ -14,10 +14,8 @@ from earthkit.data import from_source from earthkit.data.core.metadata import RawMetadata -from earthkit.data.readers.grib.metadata import GribFieldMetadata -from earthkit.data.readers.grib.metadata import StandAloneGribMetadata -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.readers.grib.metadata import GribFieldMetadata, StandAloneGribMetadata +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file @pytest.mark.parametrize( diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index 6386fef8d..e861f860b 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -14,8 +14,7 @@ import pytest from earthkit.data import settings -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file +from earthkit.data.core.temporary import temp_directory, temp_file # TODO: remove all these tests when settings are removed diff --git a/tests/data/gridspec/reduced_gg.yaml b/tests/data/gridspec/reduced_gg.yaml index 4b3b336b1..ea31ba359 100644 --- a/tests/data/gridspec/reduced_gg.yaml +++ b/tests/data/gridspec/reduced_gg.yaml @@ -1,10 +1,10 @@ - file: reduced_gg/t_reduced_gg_O320_west.grib1 gridspec: area: - - 11.944 - - -80.0 - - -9.977 - - -70.0 + - 11.944 + - -80.0 + - -9.977 + - -70.0 grid: O320 i_scans_negatively: 0 j_points_consecutive: 0 @@ -27,10 +27,10 @@ - file: reduced_gg/t_reduced_gg_O128_global.grib1 gridspec: area: - - 89.463 - - 0.0 - - -89.463 - - 359.319 + - 89.463 + - 0.0 + - -89.463 + - 359.319 grid: O128 i_scans_negatively: 0 j_points_consecutive: 0 @@ -53,10 +53,10 @@ - file: reduced_gg/t_reduced_gg_O1280_subarea.grib1 gridspec: area: - - 44.254 - - 22.269 - - 41.23 - - 28.692 + - 44.254 + - 22.269 + - 41.23 + - 28.692 grid: O1280 i_scans_negatively: 0 j_points_consecutive: 0 @@ -79,10 +79,10 @@ - file: reduced_gg/t_reduced_gg_N128_global.grib1 gridspec: area: - - 89.463 - - 0.0 - - -89.463 - - 359.297 + - 89.463 + - 0.0 + - -89.463 + - 359.297 grid: F128 i_scans_negatively: 0 j_points_consecutive: 0 @@ -108,10 +108,10 @@ - file: reduced_gg/PCE_I2D_reduced_gg_O1280_57961000_230000000_24991000_300000000.grib2 gridspec: area: - - 57.961 - - 230.0 - - 24.991 - - 300.0 + - 57.961 + - 230.0 + - 24.991 + - 300.0 grid: O1280 i_scans_negatively: 0 j_points_consecutive: 0 diff --git a/tests/data/gridspec/reduced_ll.yaml b/tests/data/gridspec/reduced_ll.yaml index 1a7b5397f..1890cfe21 100644 --- a/tests/data/gridspec/reduced_ll.yaml +++ b/tests/data/gridspec/reduced_ll.yaml @@ -1,10 +1,10 @@ - file: reduced_ll/swh_rll_global.grib1 gridspec: area: - - 90.0 - - 0.0 - - -90.0 - - 359.875 + - 90.0 + - 0.0 + - -90.0 + - 359.875 grid: 0.125 i_scans_negatively: 0 j_points_consecutive: 0 diff --git a/tests/data/gridspec/reduced_rotated_gg.yaml b/tests/data/gridspec/reduced_rotated_gg.yaml index 4643dfc70..b2f544d87 100644 --- a/tests/data/gridspec/reduced_rotated_gg.yaml +++ b/tests/data/gridspec/reduced_rotated_gg.yaml @@ -2,17 +2,17 @@ gridspec: angle_of_rotation: 0.0 area: - - 88.573 - - 0.0 - - -88.573 - - 358.125 + - 88.573 + - 0.0 + - -88.573 + - 358.125 grid: N48 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - 40.0 - - 20.0 + - 40.0 + - 20.0 type: reduced_gg metadata: N: 48 diff --git a/tests/data/gridspec/regular_gg.yaml b/tests/data/gridspec/regular_gg.yaml index b21c4f5c3..d0ecd5cf3 100644 --- a/tests/data/gridspec/regular_gg.yaml +++ b/tests/data/gridspec/regular_gg.yaml @@ -1,10 +1,10 @@ - file: regular_gg/t_regular_gg_F48_global.grib1 gridspec: area: - - 88.573 - - 0.0 - - -88.573 - - 358.125 + - 88.573 + - 0.0 + - -88.573 + - 358.125 grid: F48 i_scans_negatively: 0 j_points_consecutive: 0 diff --git a/tests/data/gridspec/regular_ll.yaml b/tests/data/gridspec/regular_ll.yaml index 4641ecca6..eae952997 100644 --- a/tests/data/gridspec/regular_ll.yaml +++ b/tests/data/gridspec/regular_ll.yaml @@ -1,13 +1,13 @@ - file: regular_ll/wrf_swh_aegean_ll_jscanpos.grib1 gridspec: area: - - 41.188 - - 22.0 - - 34.521 - - 30.0 + - 41.188 + - 22.0 + - 34.521 + - 30.0 grid: - - 0.083 - - 0.083 + - 0.083 + - 0.083 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 1 @@ -30,13 +30,13 @@ - file: regular_ll/t_global_0_360_5x5.grib2 gridspec: area: - - 90.0 - - 0.0 - - -90.0 - - 355.0 + - 90.0 + - 0.0 + - -90.0 + - 355.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -59,13 +59,13 @@ - file: regular_ll/t_75_-60_10_40_5x5.grib2 gridspec: area: - - 75.0 - - 300.0 - - 10.0 - - 40.0 + - 75.0 + - 300.0 + - 10.0 + - 40.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -88,13 +88,13 @@ - file: regular_ll/t_global_shifted_1x1.grib2 gridspec: area: - - 89.5 - - 0.5 - - -89.5 - - 359.5 + - 89.5 + - 0.5 + - -89.5 + - 359.5 grid: - - 1.0 - - 1.0 + - 1.0 + - 1.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -117,13 +117,13 @@ - file: regular_ll/t_75_60_10_-20_5x5.grib1 gridspec: area: - - 75.0 - - 60.0 - - 10.0 - - 340.0 + - 75.0 + - 60.0 + - 10.0 + - 340.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -146,13 +146,13 @@ - file: regular_ll/t_global_-180_180_5x5.grib1 gridspec: area: - - 90.0 - - -180.0 - - -90.0 - - 175.0 + - 90.0 + - -180.0 + - -90.0 + - 175.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -175,13 +175,13 @@ - file: regular_ll/t_global_0_360_5x5.grib1 gridspec: area: - - 90.0 - - 0.0 - - -90.0 - - 355.0 + - 90.0 + - 0.0 + - -90.0 + - 355.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -204,13 +204,13 @@ - file: regular_ll/wind_uk_ll_jscanpos_jcons.grib1 gridspec: area: - - 62.333 - - -11.938 - - 48.0 - - 13.063 + - 62.333 + - -11.938 + - 48.0 + - 13.063 grid: - - 0.125 - - 0.083 + - 0.125 + - 0.083 i_scans_negatively: 0 j_points_consecutive: 1 j_scans_positively: 1 @@ -233,13 +233,13 @@ - file: regular_ll/t_global_shifted_1x1.grib1 gridspec: area: - - 89.5 - - 0.5 - - -89.5 - - 359.5 + - 89.5 + - 0.5 + - -89.5 + - 359.5 grid: - - 1.0 - - 1.0 + - 1.0 + - 1.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -262,13 +262,13 @@ - file: regular_ll/t_75_-60_10_40_5x5.grib1 gridspec: area: - - 75.0 - - -60.0 - - 10.0 - - 40.0 + - 75.0 + - -60.0 + - 10.0 + - 40.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -291,13 +291,13 @@ - file: regular_ll/t_75_60_10_-20_5x5.grib2 gridspec: area: - - 75.0 - - 60.0 - - 10.0 - - 340.0 + - 75.0 + - 60.0 + - 10.0 + - 340.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 @@ -320,13 +320,13 @@ - file: regular_ll/t_global_-180_180_5x5.grib2 gridspec: area: - - 90.0 - - 180.0 - - -90.0 - - 175.0 + - 90.0 + - 180.0 + - -90.0 + - 175.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 diff --git a/tests/data/gridspec/rotated_gg.yaml b/tests/data/gridspec/rotated_gg.yaml index 73b9dfe54..3498d6d42 100644 --- a/tests/data/gridspec/rotated_gg.yaml +++ b/tests/data/gridspec/rotated_gg.yaml @@ -2,17 +2,17 @@ gridspec: angle_of_rotation: 0.0 area: - - 88.573 - - 0.0 - - -88.573 - - 358.125 + - 88.573 + - 0.0 + - -88.573 + - 358.125 grid: F48 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - 40.0 - - 20.0 + - 40.0 + - 20.0 type: regular_gg metadata: N: 48 diff --git a/tests/data/gridspec/rotated_ll.yaml b/tests/data/gridspec/rotated_ll.yaml index c4ff70a2b..3cd727bb5 100644 --- a/tests/data/gridspec/rotated_ll.yaml +++ b/tests/data/gridspec/rotated_ll.yaml @@ -2,19 +2,19 @@ gridspec: angle_of_rotation: 0.0 area: - - 90.0 - - 0.0 - - -90.0 - - 355.0 + - 90.0 + - 0.0 + - -90.0 + - 355.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - -24.0 - - 10.0 + - -24.0 + - 10.0 type: regular_ll metadata: Ni: 72 @@ -38,19 +38,19 @@ gridspec: angle_of_rotation: 0.0 area: - - 90.0 - - 180.0 - - -90.0 - - 175.0 + - 90.0 + - 180.0 + - -90.0 + - 175.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - -24.0 - - 10.0 + - -24.0 + - 10.0 type: regular_ll metadata: Ni: 72 @@ -74,19 +74,19 @@ gridspec: angle_of_rotation: 0.0 area: - - 5.5 - - 356.2 - - -8.5 - - 7.7 + - 5.5 + - 356.2 + - -8.5 + - 7.7 grid: - - 0.02 - - 0.02 + - 0.02 + - 0.02 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 1 rotation: - - -47.0 - - 10.0 + - -47.0 + - 10.0 type: regular_ll metadata: Ni: 576 @@ -110,19 +110,19 @@ gridspec: angle_of_rotation: 0.0 area: - - 90.0 - - 0.0 - - -90.0 - - 355.0 + - 90.0 + - 0.0 + - -90.0 + - 355.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - -24.0 - - 10.0 + - -24.0 + - 10.0 type: regular_ll metadata: Ni: 72 @@ -146,19 +146,19 @@ gridspec: angle_of_rotation: 0.0 area: - - 90.0 - - -180.0 - - -90.0 - - 175.0 + - 90.0 + - -180.0 + - -90.0 + - 175.0 grid: - - 5.0 - - 5.0 + - 5.0 + - 5.0 i_scans_negatively: 0 j_points_consecutive: 0 j_scans_positively: 0 rotation: - - -24.0 - - 10.0 + - -24.0 + - 10.0 type: regular_ll metadata: Ni: 72 diff --git a/tests/data/xr_engine/xr_grid.yaml b/tests/data/xr_engine/xr_grid.yaml index 0bb638dc8..ca284920b 100644 --- a/tests/data/xr_engine/xr_grid.yaml +++ b/tests/data/xr_engine/xr_grid.yaml @@ -4,11 +4,11 @@ longitude: 36 coords: latitude: - - latitude - - 19 + - latitude + - 19 longitude: - - longitude - - 36 + - longitude + - 36 distinct_ll: true - file: regular_ll.grib2 dims: @@ -16,11 +16,11 @@ longitude: 36 coords: latitude: - - latitude - - 19 + - latitude + - 19 longitude: - - longitude - - 36 + - longitude + - 36 distinct_ll: true - file: regular_ll_subarea.grib1 dims: @@ -28,11 +28,11 @@ longitude: 5 coords: latitude: - - latitude - - 4 + - latitude + - 4 longitude: - - longitude - - 5 + - longitude + - 5 distinct_ll: true - file: regular_ll_single_point.grib1 dims: @@ -40,11 +40,11 @@ longitude: 1 coords: latitude: - - latitude - - 1 + - latitude + - 1 longitude: - - longitude - - 1 + - longitude + - 1 distinct_ll: true - file: rotated_ll.grib1 dims: @@ -52,15 +52,15 @@ x: 36 coords: latitude: - - - y - - x - - - 19 - - 36 + - - y + - x + - - 19 + - 36 longitude: - - - y - - x - - - 19 - - 36 + - - y + - x + - - 19 + - 36 distinct_ll: false - file: rotated_ll_subarea.grib1 dims: @@ -68,59 +68,59 @@ x: 5 coords: latitude: - - - y - - x - - - 5 - - 5 + - - y + - x + - - 5 + - 5 longitude: - - - y - - x - - - 5 - - 5 + - - y + - x + - - 5 + - 5 distinct_ll: false - file: reduced_gg_N32.grib1 dims: values: 6114 coords: latitude: - - values - - 6114 + - values + - 6114 longitude: - - values - - 6114 + - values + - 6114 distinct_ll: false - file: reduced_gg_N32.grib2 dims: values: 6114 coords: latitude: - - values - - 6114 + - values + - 6114 longitude: - - values - - 6114 + - values + - 6114 distinct_ll: false - file: reduced_gg_O32.grib1 dims: values: 5248 coords: latitude: - - values - - 5248 + - values + - 5248 longitude: - - values - - 5248 + - values + - 5248 distinct_ll: false - file: reduced_gg_O32.grib2 dims: values: 5248 coords: latitude: - - values - - 5248 + - values + - 5248 longitude: - - values - - 5248 + - values + - 5248 distinct_ll: false - file: regular_gg_F16.grib1 dims: @@ -128,77 +128,77 @@ longitude: 64 coords: latitude: - - latitude - - 32 + - latitude + - 32 longitude: - - longitude - - 64 + - longitude + - 64 distinct_ll: true - file: reduced_rotated_gg_N32.grib1 dims: values: 6114 coords: latitude: - - values - - 6114 + - values + - 6114 longitude: - - values - - 6114 + - values + - 6114 distinct_ll: false - file: reduced_rotated_gg_O32.grib1 dims: values: 5248 coords: latitude: - - values - - 5248 + - values + - 5248 longitude: - - values - - 5248 + - values + - 5248 distinct_ll: false - file: reduced_rotated_gg_subarea_N32.grib1 dims: values: 225 coords: latitude: - - values - - 225 + - values + - 225 longitude: - - values - - 225 + - values + - 225 distinct_ll: false - file: reduced_rotated_gg_subarea_O32.grib1 dims: values: 225 coords: latitude: - - values - - 225 + - values + - 225 longitude: - - values - - 225 + - values + - 225 distinct_ll: false - file: healpix_H8.grib2 dims: values: 768 coords: latitude: - - values - - 768 + - values + - 768 longitude: - - values - - 768 + - values + - 768 distinct_ll: false - file: healpix_H8_nested.grib2 dims: values: 768 coords: latitude: - - values - - 768 + - values + - 768 longitude: - - values - - 768 + - values + - 768 distinct_ll: false - file: sh_t32.grib1 dims: diff --git a/tests/documentation/test_notebooks.py b/tests/documentation/test_notebooks.py index b8c758e77..e52009ef8 100644 --- a/tests/documentation/test_notebooks.py +++ b/tests/documentation/test_notebooks.py @@ -14,11 +14,10 @@ import sys import pytest - -from earthkit.data.testing import MISSING -from earthkit.data.testing import earthkit_file from earthkit.utils.array.testing.testing import NO_TORCH +from earthkit.data.testing import MISSING, earthkit_file + # See https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ diff --git a/tests/environment-unit-tests.yml b/tests/environment-unit-tests.yml index 3b5f25a85..0297b177a 100644 --- a/tests/environment-unit-tests.yml +++ b/tests/environment-unit-tests.yml @@ -1,65 +1,65 @@ name: earthkit-data channels: - - conda-forge - - nodefaults +- conda-forge +- nodefaults dependencies: - - eccodes=>2.35.0 - - python-eccodes>=1.7.0 - - pip - - numpy - - pandas - - xarray>=0.19.0 - - cartopy - - dask - - netcdf4 - - cfgrib>=0.9.10.1 - - pdbufr>=0.11.0 - - pyodc - - filelock - - pyyaml - - entrypoints - - jupyterlab - - ecmwf-api-client>=1.6.1 - - cdsapi>=0.7.2 - - hda>2.22 - - jsonschema - - pip: - - multiurl>=0.3.3 - - pyfdb>=0.1.0 - - ecmwf-opendata>=0.3.3 - - polytope-client>=0.7.4 - - git+https://github.com/ecmwf/earthkit-data-demo-source - - covjsonkit>=0.2.2 - - earthkit-geo>=0.2.0 - - earthkit-utils>=0.2.0 - - tqdm>=4.63.0 - - lru-dict - - markdown - - aws-requests-auth - - botocore - - make - - mypy - - myst-parser - - pre-commit - - pydata-sphinx-theme - - pyfakefs - - pytest - - pytest-cov - - pytest-forked - - pytest-reraise - - pytest-timeout - - sphinx - - sphinx-autoapi - - sphinx_rtd_theme - - sphinxcontrib-apidoc - - sphinx-issues - - sphinx-tabs - - sphinx-copybutton - - nbformat - - nbconvert - - nbsphinx - - ipykernel - - geopandas - - deprecation - - ncdata - - scitools-iris +- eccodes=>2.35.0 +- python-eccodes>=1.7.0 +- pip +- numpy +- pandas +- xarray>=0.19.0 +- cartopy +- dask +- netcdf4 +- cfgrib>=0.9.10.1 +- pdbufr>=0.11.0 +- pyodc +- filelock +- pyyaml +- entrypoints +- jupyterlab +- ecmwf-api-client>=1.6.1 +- cdsapi>=0.7.2 +- hda>2.22 +- jsonschema +- pip: + - multiurl>=0.3.3 + - pyfdb>=0.1.0 + - ecmwf-opendata>=0.3.3 + - polytope-client>=0.7.4 + - git+https://github.com/ecmwf/earthkit-data-demo-source + - covjsonkit>=0.2.2 + - earthkit-geo>=0.2.0 + - earthkit-utils>=0.2.0 +- tqdm>=4.63.0 +- lru-dict +- markdown +- aws-requests-auth +- botocore +- make +- mypy +- myst-parser +- pre-commit +- pydata-sphinx-theme +- pyfakefs +- pytest +- pytest-cov +- pytest-forked +- pytest-reraise +- pytest-timeout +- sphinx +- sphinx-autoapi +- sphinx_rtd_theme +- sphinxcontrib-apidoc +- sphinx-issues +- sphinx-tabs +- sphinx-copybutton +- nbformat +- nbconvert +- nbsphinx +- ipykernel +- geopandas +- deprecation +- ncdata +- scitools-iris diff --git a/tests/forcings/conftest.py b/tests/forcings/conftest.py new file mode 100644 index 000000000..e669d95eb --- /dev/null +++ b/tests/forcings/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/tests/forcings/test_forcings_metadata.py b/tests/forcings/test_forcings_metadata.py index 6fdafd105..ee1ca0038 100644 --- a/tests/forcings/test_forcings_metadata.py +++ b/tests/forcings/test_forcings_metadata.py @@ -10,14 +10,9 @@ # import datetime -import os -import sys import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import load_forcings_fs # noqa: E402 +from forcings_fixtures import load_forcings_fs @pytest.mark.parametrize("input_data", ["grib", "latlon"]) diff --git a/tests/forcings/test_forcings_proc.py b/tests/forcings/test_forcings_proc.py index 0bdb2ae60..723091903 100644 --- a/tests/forcings/test_forcings_proc.py +++ b/tests/forcings/test_forcings_proc.py @@ -10,19 +10,17 @@ # import os -import sys import numpy as np import pytest import yaml +from forcings_fixtures import ( + all_params, + load_forcings_fs, +) from earthkit.data.testing import earthkit_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import all_params # noqa: E402 -from forcings_fixtures import load_forcings_fs # noqa: E402 - @pytest.mark.parametrize("input_data", ["grib", "latlon"]) def _build_proc_ref(input_data): diff --git a/tests/forcings/test_forcings_sel.py b/tests/forcings/test_forcings_sel.py index 1785f5bec..51ef53000 100644 --- a/tests/forcings/test_forcings_sel.py +++ b/tests/forcings/test_forcings_sel.py @@ -10,14 +10,8 @@ # -import os -import sys - import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import load_forcings_fs # noqa: E402 +from forcings_fixtures import load_forcings_fs @pytest.mark.parametrize("input_data", ["grib", "latlon"]) @@ -59,12 +53,10 @@ def test_forcings_sel_single_file_1(input_data, params, expected_meta): def test_forcings_sel_single_file_as_dict(input_data): ds, _ = load_forcings_fs(input_data=input_data) - g = ds.sel( - { - "param": "sin_longitude", - "valid_datetime": ["2020-05-14T06:00:00", "2020-05-13T18:00:00"], - } - ) + g = ds.sel({ + "param": "sin_longitude", + "valid_datetime": ["2020-05-14T06:00:00", "2020-05-13T18:00:00"], + }) assert len(g) == 2 assert g.metadata(["param", "valid_datetime"]) == [ diff --git a/tests/forcings/test_forcings_slice.py b/tests/forcings/test_forcings_slice.py index 1ae4b0178..24f65e6b0 100644 --- a/tests/forcings/test_forcings_slice.py +++ b/tests/forcings/test_forcings_slice.py @@ -9,15 +9,10 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import load_forcings_fs # noqa: E402 +from forcings_fixtures import load_forcings_fs @pytest.mark.parametrize("input_data", ["grib", "latlon"]) diff --git a/tests/forcings/test_forcings_source.py b/tests/forcings/test_forcings_source.py index 51261d744..d4bcf84b4 100644 --- a/tests/forcings/test_forcings_source.py +++ b/tests/forcings/test_forcings_source.py @@ -11,18 +11,12 @@ import datetime import itertools -import os -import sys import pytest +from forcings_fixtures import all_params from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import all_params # noqa: E402 +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file def test_forcings_source_1(): diff --git a/tests/forcings/test_forcings_values.py b/tests/forcings/test_forcings_values.py index cc38c9cd1..ff373def0 100644 --- a/tests/forcings/test_forcings_values.py +++ b/tests/forcings/test_forcings_values.py @@ -10,15 +10,9 @@ # -import os -import sys - import numpy as np import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from forcings_fixtures import load_forcings_fs # noqa: E402 +from forcings_fixtures import load_forcings_fs def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3): diff --git a/tests/grib/conftest.py b/tests/grib/conftest.py new file mode 100644 index 000000000..e669d95eb --- /dev/null +++ b/tests/grib/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/tests/grib/grib_fixtures.py b/tests/grib/grib_fixtures.py index 2ac504dc2..e56418d3e 100644 --- a/tests/grib/grib_fixtures.py +++ b/tests/grib/grib_fixtures.py @@ -11,10 +11,7 @@ from earthkit.data import from_source -from earthkit.data.testing import ARRAY_BACKENDS -from earthkit.data.testing import ArrayBackend -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import ARRAY_BACKENDS, ArrayBackend, earthkit_examples_file, earthkit_test_data_file def load_array_fieldlist(path, array_namespace=None, device=None, dtype=None, **kwargs): diff --git a/tests/grib/test_grib_backend.py b/tests/grib/test_grib_backend.py index 540b3fa0f..95b1c8fff 100644 --- a/tests/grib/test_grib_backend.py +++ b/tests/grib/test_grib_backend.py @@ -11,13 +11,11 @@ import numpy as np import pytest +from earthkit.utils.array import array_namespace as eku_array_namespace +from earthkit.utils.array.testing.testing import NO_CUPY, NO_TORCH -from earthkit.data import FieldList -from earthkit.data import from_source +from earthkit.data import FieldList, from_source from earthkit.data.testing import earthkit_examples_file -from earthkit.utils.array import array_namespace as eku_array_namespace -from earthkit.utils.array.testing.testing import NO_CUPY -from earthkit.utils.array.testing.testing import NO_TORCH @pytest.mark.parametrize("_kwargs", [{}, {"array_backend": "numpy"}, {"array_namespace": "numpy"}]) diff --git a/tests/grib/test_grib_cache.py b/tests/grib/test_grib_cache.py index d9454894d..a88fad5bf 100644 --- a/tests/grib/test_grib_cache.py +++ b/tests/grib/test_grib_cache.py @@ -9,21 +9,15 @@ # nor does it submit to any jurisdiction. # -import os import pickle -import sys import pytest +from grib_fixtures import load_grib_data -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import config, from_source from earthkit.data.testing import earthkit_examples_file from earthkit.data.utils.diag import field_cache_diag -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import load_grib_data # noqa: E402 - FIELD_NUM = 18 MD_ITEM_NUM = 7 @@ -68,14 +62,12 @@ def _check_diag(diag, ref): @pytest.mark.parametrize("serialise", [True, False]) def test_grib_cache_basic_file_patched(handle_cache_size, serialise, patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": handle_cache_size, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": handle_cache_size, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) if serialise: @@ -149,17 +141,16 @@ def test_grib_cache_basic_file_patched(handle_cache_size, serialise, patch_metad def test_grib_cache_basic_file_non_patched(): - """This test is the same as test_grib_cache_basic but without the patch_metadata_cache fixture. - So metadata cache hits and misses are not counted.""" - - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + """The test is the same as test_grib_cache_basic but + without the patch_metadata_cache fixture. + So metadata cache hits and misses are not counted. + """ + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -224,14 +215,12 @@ def test_grib_cache_basic_file_non_patched(): @pytest.mark.parametrize("fl_type", ["file", "array", "memory"]) def test_grib_cache_basic_metadata_patched(serialise, fl_type, patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds, _ = load_grib_data("tuv_pl.grib", fl_type) if serialise: @@ -290,14 +279,12 @@ def test_grib_cache_basic_metadata_patched(serialise, fl_type, patch_metadata_ca def test_grib_cache_options_1(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "temporary", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "temporary", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -382,14 +369,12 @@ def test_grib_cache_options_1(patch_metadata_cache): def test_grib_cache_options_2(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "persistent", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "persistent", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -476,14 +461,12 @@ def test_grib_cache_options_2(patch_metadata_cache): def test_grib_cache_options_3(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -568,14 +551,12 @@ def test_grib_cache_options_3(patch_metadata_cache): def test_grib_cache_options_4(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "temporary", - "grib-handle-policy": "temporary", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "temporary", + "grib-handle-policy": "temporary", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -665,14 +646,12 @@ def test_grib_cache_options_4(patch_metadata_cache): def test_grib_cache_options_5(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "temporary", - "grib-handle-policy": "persistent", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "temporary", + "grib-handle-policy": "persistent", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -764,14 +743,12 @@ def test_grib_cache_options_5(patch_metadata_cache): def test_grib_cache_options_6(patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "temporary", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): + with config.temporary({ + "grib-field-policy": "temporary", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): ds = from_source("file", earthkit_examples_file("tuv_pl.grib")) assert len(ds) == FIELD_NUM @@ -897,15 +874,12 @@ def test_grib_cache_file_use_kwargs_2(): @pytest.mark.parametrize("fl_type", ["file", "array", "memory"]) def test_grib_cache_metadata_use_kwargs_1(fl_type, patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": False, - } - ): - + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": False, + }): _kwargs = { "use_grib_metadata_cache": True, } @@ -939,15 +913,12 @@ def test_grib_cache_metadata_use_kwargs_1(fl_type, patch_metadata_cache): @pytest.mark.parametrize("fl_type", ["file", "array", "memory"]) def test_grib_cache_metadata_use_kwargs_2(fl_type, patch_metadata_cache): - with config.temporary( - { - "grib-field-policy": "persistent", - "grib-handle-policy": "cache", - "grib-handle-cache-size": 1, - "use-grib-metadata-cache": True, - } - ): - + with config.temporary({ + "grib-field-policy": "persistent", + "grib-handle-policy": "cache", + "grib-handle-cache-size": 1, + "use-grib-metadata-cache": True, + }): _kwargs = { "use_grib_metadata_cache": False, } diff --git a/tests/grib/test_grib_convert.py b/tests/grib/test_grib_convert.py index 602ed8aaf..80fd39519 100644 --- a/tests/grib/test_grib_convert.py +++ b/tests/grib/test_grib_convert.py @@ -9,16 +9,13 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_NUMPY # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from grib_fixtures import ( + FL_NUMPY, + load_grib_data, +) @pytest.mark.parametrize("fl_type", FL_NUMPY) diff --git a/tests/grib/test_grib_copy.py b/tests/grib/test_grib_copy.py index 265ace815..30b072f3b 100644 --- a/tests/grib/test_grib_copy.py +++ b/tests/grib/test_grib_copy.py @@ -9,22 +9,15 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from grib_fixtures import load_grib_data -from earthkit.data import FieldList -from earthkit.data import from_source +from earthkit.data import FieldList, from_source from earthkit.data.core.temporary import temp_file from earthkit.data.sources.array_list import ArrayField -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import write_to_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import load_grib_data # noqa: E402 +from earthkit.data.testing import WRITE_TO_FILE_METHODS, write_to_file # @pytest.mark.parametrize("fl_type", ["file", "array", "memory"]) diff --git a/tests/grib/test_grib_cube.py b/tests/grib/test_grib_cube.py index b46c74df7..6de0e1aa8 100644 --- a/tests/grib/test_grib_cube.py +++ b/tests/grib/test_grib_cube.py @@ -13,8 +13,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file def test_grib_cube_core(): diff --git a/tests/grib/test_grib_file_stream.py b/tests/grib/test_grib_file_stream.py index 7ef9ef217..142efd3da 100644 --- a/tests/grib/test_grib_file_stream.py +++ b/tests/grib/test_grib_file_stream.py @@ -15,14 +15,15 @@ import pytest from earthkit.data import from_source -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file +from earthkit.data.core.temporary import temp_directory, temp_file from earthkit.data.sources.stream import StreamFieldList -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import make_tgz -from earthkit.data.testing import write_to_file +from earthkit.data.testing import ( + WRITE_TO_FILE_METHODS, + earthkit_examples_file, + earthkit_remote_test_data_file, + make_tgz, + write_to_file, +) def repeat_list_items(items, count): @@ -139,16 +140,14 @@ def test_grib_file_stream_in_memory(): # data assert ds.to_numpy().shape == expected_shape - ref = np.array( - [ - 272.56417847, - -6.28688049, - 7.83348083, - 272.53916931, - -4.89837646, - 8.66096497, - ] - ) + ref = np.array([ + 272.56417847, + -6.28688049, + 7.83348083, + 272.53916931, + -4.89837646, + 8.66096497, + ]) vals = ds.to_numpy()[:, 0, 0] assert np.allclose(vals, ref) diff --git a/tests/grib/test_grib_geography.py b/tests/grib/test_grib_geography.py index fe75c8348..71a8892ec 100644 --- a/tests/grib/test_grib_geography.py +++ b/tests/grib/test_grib_geography.py @@ -9,26 +9,19 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from earthkit.utils.array import convert as array_convert +from grib_fixtures import ( + FL_NUMPY, + FL_TYPES, + load_grib_data, +) import earthkit.data -from earthkit.data.testing import NO_GEO -from earthkit.data.testing import check_array -from earthkit.data.testing import check_array_type -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_GEO, check_array, check_array_type, earthkit_examples_file, earthkit_test_data_file from earthkit.data.utils import projections -from earthkit.utils.array import convert as array_convert - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_NUMPY # noqa: E402 -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 @pytest.mark.parametrize("fl_type", FL_TYPES) @@ -361,15 +354,16 @@ def test_grib_mars_grid(path, expected_value): @pytest.mark.skipif(NO_GEO, reason="No earthkit-geo support") def test_grib_grid_points_rotated_ll(): - """The""" + """The.""" ds = earthkit.data.from_source("file", earthkit_test_data_file("rotated_wind_20x20.grib")) # grid points res = ds[0].grid_points() ref1 = np.array([30.0, 29.351052, 27.504876, 24.734374]), np.array([140.0, 136.09296, 132.770576, 130.469424]) - ref2 = np.array([-17.968188, -14.787578, -12.22927, -10.573044]), np.array( - [-50.356844, -48.94784, -46.558096, -43.46374] + ref2 = ( + np.array([-17.968188, -14.787578, -12.22927, -10.573044]), + np.array([-50.356844, -48.94784, -46.558096, -43.46374]), ) assert np.allclose(res[0][:4], ref1[0]) @@ -409,13 +403,15 @@ def test_grib_grid_points_rotated_rgg(): res = ds[0].grid_points_unrotated() # front - ref1 = np.array([26.510768, 26.51076943, 26.5107701, 26.51076846]), np.array( - [1.28492181e-15, 2.81250046e00, 5.62500163e00, 8.43749805e00] + ref1 = ( + np.array([26.510768, 26.51076943, 26.5107701, 26.51076846]), + np.array([1.28492181e-15, 2.81250046e00, 5.62500163e00, 8.43749805e00]), ) # back - ref2 = np.array([-12.55775535, -12.55775697, -12.55775699, -12.5577565]), np.array( - [30.93749931, 33.75000128, 36.56250084, 39.37500023] + ref2 = ( + np.array([-12.55775535, -12.55775697, -12.55775699, -12.5577565]), + np.array([30.93749931, 33.75000128, 36.56250084, 39.37500023]), ) assert np.allclose(res[0][:4], ref1[0]) diff --git a/tests/grib/test_grib_gridspec.py b/tests/grib/test_grib_gridspec.py index 057d055a7..7ccd104b7 100644 --- a/tests/grib/test_grib_gridspec.py +++ b/tests/grib/test_grib_gridspec.py @@ -14,16 +14,16 @@ import pytest import yaml -from earthkit.data import FieldList -from earthkit.data import from_source +from earthkit.data import FieldList, from_source from earthkit.data.core.gridspec import GridSpec from earthkit.data.core.temporary import temp_file -from earthkit.data.readers.grib.gridspec import GridSpecConverter -from earthkit.data.readers.grib.gridspec import make_gridspec -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import earthkit_test_data_file -from earthkit.data.testing import write_to_file +from earthkit.data.readers.grib.gridspec import GridSpecConverter, make_gridspec +from earthkit.data.testing import ( + WRITE_TO_FILE_METHODS, + earthkit_remote_test_data_file, + earthkit_test_data_file, + write_to_file, +) SUPPORTED_GRID_TYPES = [ "sh", @@ -148,15 +148,13 @@ def test_grib_metadata_from_gridspec_valid(metadata, gridspec, name): @pytest.mark.parametrize( "metadata,gridspec,name", - gridspec_list( - [ - "sh", - "reduced_ll", - "regular_gg", - "reduced_gg", - "healpix", - ] - ), + gridspec_list([ + "sh", + "reduced_ll", + "regular_gg", + "reduced_gg", + "healpix", + ]), ) def test_grib_metadata_from_gridspec_invalid(metadata, gridspec, name): if name in [ diff --git a/tests/grib/test_grib_headers_only.py b/tests/grib/test_grib_headers_only.py index 4061361e5..e8ef0840a 100644 --- a/tests/grib/test_grib_headers_only.py +++ b/tests/grib/test_grib_headers_only.py @@ -17,13 +17,9 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file -from earthkit.data.readers.grib.metadata import RestrictedGribMetadata -from earthkit.data.readers.grib.metadata import StandAloneGribMetadata -from earthkit.data.readers.grib.metadata import WrappedMetadata +from earthkit.data.readers.grib.metadata import RestrictedGribMetadata, StandAloneGribMetadata, WrappedMetadata from earthkit.data.readers.grib.output import new_grib_output -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_test_data_file, earthkit_test_data_file from earthkit.data.utils import ensure_iterable diff --git a/tests/grib/test_grib_inidces.py b/tests/grib/test_grib_inidces.py index feb9a5704..f3cad41de 100644 --- a/tests/grib/test_grib_inidces.py +++ b/tests/grib/test_grib_inidces.py @@ -9,15 +9,12 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from grib_fixtures import ( + FL_TYPES, + load_grib_data, +) @pytest.mark.parametrize("fl_type", FL_TYPES) diff --git a/tests/grib/test_grib_iter.py b/tests/grib/test_grib_iter.py index 418d1e1d3..3a2eed02b 100644 --- a/tests/grib/test_grib_iter.py +++ b/tests/grib/test_grib_iter.py @@ -10,19 +10,15 @@ # -import os -import sys - import pytest +from grib_fixtures import ( + FL_ARRAYS, + load_grib_data, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_examples_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_ARRAYS # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 - @pytest.mark.parametrize("fl_type", FL_ARRAYS) @pytest.mark.parametrize("group", ["param"]) diff --git a/tests/grib/test_grib_memory.py b/tests/grib/test_grib_memory.py index fa6d98f1f..d3303ef3a 100644 --- a/tests/grib/test_grib_memory.py +++ b/tests/grib/test_grib_memory.py @@ -13,10 +13,7 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file -from earthkit.data.testing import write_to_file +from earthkit.data.testing import WRITE_TO_FILE_METHODS, earthkit_examples_file, earthkit_test_data_file, write_to_file def test_grib_from_memory_single(): diff --git a/tests/grib/test_grib_metadata.py b/tests/grib/test_grib_metadata.py index f89682d85..96235a943 100644 --- a/tests/grib/test_grib_metadata.py +++ b/tests/grib/test_grib_metadata.py @@ -10,21 +10,17 @@ # import datetime -import os -import sys import numpy as np import pytest +from grib_fixtures import ( + FL_FILE, + FL_TYPES, + load_grib_data, +) from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_FILE # noqa: E402 -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_test_data_file def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3): diff --git a/tests/grib/test_grib_order_by.py b/tests/grib/test_grib_order_by.py index 5c71c816c..54e97fe58 100644 --- a/tests/grib/test_grib_order_by.py +++ b/tests/grib/test_grib_order_by.py @@ -9,18 +9,15 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from grib_fixtures import ( + FL_TYPES, + load_grib_data, +) from earthkit.data import from_source -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 - # @pytest.mark.skipif(("GITHUB_WORKFLOW" in os.environ) or True, reason="Not yet ready") @pytest.mark.parametrize("fl_type", FL_TYPES) diff --git a/tests/grib/test_grib_output.py b/tests/grib/test_grib_output.py index 554c6d529..b307795fe 100644 --- a/tests/grib/test_grib_output.py +++ b/tests/grib/test_grib_output.py @@ -16,20 +16,15 @@ import numpy as np import pytest +from grib_fixtures import ( + FL_ARRAYS, + load_grib_data, +) import earthkit.data -from earthkit.data import from_source -from earthkit.data import to_target -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import write_to_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_ARRAYS # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from earthkit.data import from_source, to_target +from earthkit.data.core.temporary import temp_directory, temp_file +from earthkit.data.testing import WRITE_TO_FILE_METHODS, earthkit_examples_file, write_to_file EPSILON = 1e-4 diff --git a/tests/grib/test_grib_sel.py b/tests/grib/test_grib_sel.py index 7fcb96d08..04b6c041f 100644 --- a/tests/grib/test_grib_sel.py +++ b/tests/grib/test_grib_sel.py @@ -9,19 +9,16 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from grib_fixtures import ( + FL_TYPES, + load_grib_data, +) from earthkit.data import from_source -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 - # @pytest.mark.skipif(("GITHUB_WORKFLOW" in os.environ) or True, reason="Not yet ready") diff --git a/tests/grib/test_grib_serialise.py b/tests/grib/test_grib_serialise.py index f9e19ff57..36af696f6 100644 --- a/tests/grib/test_grib_serialise.py +++ b/tests/grib/test_grib_serialise.py @@ -11,24 +11,19 @@ import os import pickle -import sys import numpy as np import pytest +from grib_fixtures import ( + FL_FILE, + FL_NUMPY, + load_grib_data, +) -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import config, from_source from earthkit.data.core.temporary import temp_file from earthkit.data.readers.grib.metadata import StandAloneGribMetadata -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import write_to_file - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_FILE # noqa: E402 -from grib_fixtures import FL_NUMPY # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from earthkit.data.testing import WRITE_TO_FILE_METHODS, earthkit_examples_file, write_to_file def _pickle(data, representation): diff --git a/tests/grib/test_grib_simplefieldlist.py b/tests/grib/test_grib_simplefieldlist.py index bd45c42b7..b5a1928c5 100644 --- a/tests/grib/test_grib_simplefieldlist.py +++ b/tests/grib/test_grib_simplefieldlist.py @@ -11,9 +11,7 @@ import pytest -from earthkit.data import ArrayField -from earthkit.data import SimpleFieldList -from earthkit.data import from_source +from earthkit.data import ArrayField, SimpleFieldList, from_source from earthkit.data.testing import earthkit_examples_file diff --git a/tests/grib/test_grib_slice.py b/tests/grib/test_grib_slice.py index 8749f16ea..9efb8ea1d 100644 --- a/tests/grib/test_grib_slice.py +++ b/tests/grib/test_grib_slice.py @@ -9,20 +9,17 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from grib_fixtures import ( + FL_TYPES, + load_grib_data, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_examples_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 - @pytest.mark.parametrize("fl_type", FL_TYPES) @pytest.mark.parametrize( diff --git a/tests/grib/test_grib_stream.py b/tests/grib/test_grib_stream.py index e69ab5543..acbdbc4ec 100644 --- a/tests/grib/test_grib_stream.py +++ b/tests/grib/test_grib_stream.py @@ -15,11 +15,13 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file from earthkit.data.sources.stream import StreamFieldList -from earthkit.data.testing import ARRAY_BACKENDS -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file -from earthkit.data.testing import write_to_file +from earthkit.data.testing import ( + ARRAY_BACKENDS, + WRITE_TO_FILE_METHODS, + earthkit_examples_file, + earthkit_remote_examples_file, + write_to_file, +) def repeat_list_items(items, count): @@ -232,16 +234,14 @@ def test_grib_from_stream_in_memory(): # data assert ds.to_numpy().shape == expected_shape - ref = np.array( - [ - 272.56417847, - -6.28688049, - 7.83348083, - 272.53916931, - -4.89837646, - 8.66096497, - ] - ) + ref = np.array([ + 272.56417847, + -6.28688049, + 7.83348083, + 272.53916931, + -4.89837646, + 8.66096497, + ]) vals = ds.to_numpy()[:, 0, 0] assert np.allclose(vals, ref) @@ -295,16 +295,14 @@ def test_grib_from_stream_in_memory_convert_to_numpy(convert_kwargs, expected_sh # data assert ds.to_numpy(**convert_kwargs).shape == expected_shape - ref = np.array( - [ - 272.56417847, - -6.28688049, - 7.83348083, - 272.53916931, - -4.89837646, - 8.66096497, - ] - ) + ref = np.array([ + 272.56417847, + -6.28688049, + 7.83348083, + 272.53916931, + -4.89837646, + 8.66096497, + ]) if len(expected_shape) == 3: vals = ds.to_numpy(**convert_kwargs)[:, 0, 0] diff --git a/tests/grib/test_grib_summary.py b/tests/grib/test_grib_summary.py index c52e92b2b..5aa9e3377 100644 --- a/tests/grib/test_grib_summary.py +++ b/tests/grib/test_grib_summary.py @@ -8,16 +8,13 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -import os -import sys import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_FILE # noqa: E402 -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from grib_fixtures import ( + FL_FILE, + FL_TYPES, + load_grib_data, +) @pytest.mark.parametrize("fl_type", FL_TYPES) diff --git a/tests/grib/test_grib_tensor.py b/tests/grib/test_grib_tensor.py index 3905ccfc0..4683b46bc 100644 --- a/tests/grib/test_grib_tensor.py +++ b/tests/grib/test_grib_tensor.py @@ -83,9 +83,9 @@ def test_grib_tensor_core(): cnt = 0 for par in range(2): for level in range(2): - assert ( - r[par, level].source[0].metadata(["param", "level"]) == ref_meta[cnt] - ), f"{cnt=} ref_meta={ref_meta[cnt]}" + assert r[par, level].source[0].metadata(["param", "level"]) == ref_meta[cnt], ( + f"{cnt=} ref_meta={ref_meta[cnt]}" + ) cnt += 1 # this slice is a cube @@ -103,9 +103,9 @@ def test_grib_tensor_core(): cnt = 0 for par in range(1): for level in range(6): - assert ( - r[par, level].source[0].metadata(["param", "level"]) == ref_meta[cnt] - ), f"{cnt=} ref_meta={ref_meta[cnt]}" + assert r[par, level].source[0].metadata(["param", "level"]) == ref_meta[cnt], ( + f"{cnt=} ref_meta={ref_meta[cnt]}" + ) cnt += 1 diff --git a/tests/grib/test_grib_url_stream.py b/tests/grib/test_grib_url_stream.py index d921f8feb..0fd19dece 100644 --- a/tests/grib/test_grib_url_stream.py +++ b/tests/grib/test_grib_url_stream.py @@ -15,10 +15,12 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file from earthkit.data.sources.stream import StreamFieldList -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_remote_examples_file -from earthkit.data.testing import earthkit_remote_file -from earthkit.data.testing import write_to_file +from earthkit.data.testing import ( + WRITE_TO_FILE_METHODS, + earthkit_remote_examples_file, + earthkit_remote_file, + write_to_file, +) def repeat_list_items(items, count): @@ -135,16 +137,14 @@ def test_grib_url_stream_in_memory(): # data assert ds.to_numpy().shape == expected_shape - ref = np.array( - [ - 272.56417847, - -6.28688049, - 7.83348083, - 272.53916931, - -4.89837646, - 8.66096497, - ] - ) + ref = np.array([ + 272.56417847, + -6.28688049, + 7.83348083, + 272.53916931, + -4.89837646, + 8.66096497, + ]) vals = ds.to_numpy()[:, 0, 0] assert np.allclose(vals, ref) diff --git a/tests/grib/test_grib_values.py b/tests/grib/test_grib_values.py index b4b1bd938..7d5b977bc 100644 --- a/tests/grib/test_grib_values.py +++ b/tests/grib/test_grib_values.py @@ -9,23 +9,19 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest - -from earthkit.data.testing import check_array -from earthkit.data.testing import check_array_type from earthkit.utils.array import convert as array_convert from earthkit.utils.array.convert import convert_dtype +from grib_fixtures import ( + FL_FILE, + FL_NUMPY, + FL_TYPES, + load_grib_data, +) -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from grib_fixtures import FL_FILE # noqa: E402 -from grib_fixtures import FL_NUMPY # noqa: E402 -from grib_fixtures import FL_TYPES # noqa: E402 -from grib_fixtures import load_grib_data # noqa: E402 +from earthkit.data.testing import check_array, check_array_type @pytest.mark.parametrize("fl_type", FL_TYPES) diff --git a/tests/indexing/indexing_fixtures.py b/tests/indexing/indexing_fixtures.py index 13524a4d4..a3f4e2a6a 100644 --- a/tests/indexing/indexing_fixtures.py +++ b/tests/indexing/indexing_fixtures.py @@ -12,10 +12,8 @@ import os import shutil -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.core.temporary import temp_directory, temp_file +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file TEST_GRIB_FILES = [ earthkit_examples_file(p) diff --git a/tests/indexing/test_indexing_serialisation.py b/tests/indexing/test_indexing_serialisation.py index bc0e05877..c2f128f4a 100644 --- a/tests/indexing/test_indexing_serialisation.py +++ b/tests/indexing/test_indexing_serialisation.py @@ -15,9 +15,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.utils.serialise import SERIALISATION -from earthkit.data.utils.serialise import deserialise_state -from earthkit.data.utils.serialise import serialise_state +from earthkit.data.utils.serialise import SERIALISATION, deserialise_state, serialise_state here = os.path.dirname(__file__) sys.path.insert(0, here) diff --git a/tests/lazy/test_lazy_fdb.py b/tests/lazy/test_lazy_fdb.py index 8adf1c57d..f11a3e367 100644 --- a/tests/lazy/test_lazy_fdb.py +++ b/tests/lazy/test_lazy_fdb.py @@ -17,8 +17,7 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import NO_FDB -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_FDB, earthkit_test_data_file TEST_GRIB_REQUEST = { "class": "od", @@ -99,12 +98,20 @@ def test_lazy_fdb(): assert len(t_fdb) == 16 assert len(r_fdb) == 16 - assert t.metadata(["shortName", "date", "time", "step", "levelist"]) == t_fdb.metadata( - ["shortName", "date", "time", "step", "levelist"] - ) - assert r.metadata(["shortName", "date", "time", "step", "levelist"]) == r_fdb.metadata( - ["shortName", "date", "time", "step", "levelist"] - ) + assert t.metadata(["shortName", "date", "time", "step", "levelist"]) == t_fdb.metadata([ + "shortName", + "date", + "time", + "step", + "levelist", + ]) + assert r.metadata(["shortName", "date", "time", "step", "levelist"]) == r_fdb.metadata([ + "shortName", + "date", + "time", + "step", + "levelist", + ]) assert np.allclose(t.to_numpy(), t_fdb.to_numpy().reshape(16, 19, 36)) assert np.allclose(r.to_numpy(), r_fdb.to_numpy().reshape(16, 19, 36)) diff --git a/tests/netcdf/test_netcdf_clone.py b/tests/netcdf/test_netcdf_clone.py index a6d225ae7..d49a053a0 100644 --- a/tests/netcdf/test_netcdf_clone.py +++ b/tests/netcdf/test_netcdf_clone.py @@ -9,8 +9,7 @@ # nor does it submit to any jurisdiction. # -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source def test_netcdf_clone(): diff --git a/tests/netcdf/test_netcdf_concat.py b/tests/netcdf/test_netcdf_concat.py index b5957cd49..d966b347b 100644 --- a/tests/netcdf/test_netcdf_concat.py +++ b/tests/netcdf/test_netcdf_concat.py @@ -16,8 +16,7 @@ import xarray as xr from earthkit.data import from_source -from earthkit.data.testing import earthkit_test_data_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_test_data_file, load_nc_or_xr_source # These functionalities are variations around # http://xarray.pydata.org/en/stable/user-guide/combining.html#combining-multi @@ -115,12 +114,10 @@ def test_netcdf_read_multiple_files(): import xarray as xr - target = xr.merge( - [ - xr.open_dataset(earthkit_test_data_file("era5_2t_1.nc")), - xr.open_dataset(earthkit_test_data_file("era5_2t_2.nc")), - ] - ) + target = xr.merge([ + xr.open_dataset(earthkit_test_data_file("era5_2t_1.nc")), + xr.open_dataset(earthkit_test_data_file("era5_2t_2.nc")), + ]) merged = ds.to_xarray() assert target.identical(merged) @@ -345,12 +342,10 @@ def get_hierarchy(): coord_values=dict(forecast_time=[2, 4]), ) - target = xr.merge( - [ - xr.merge([a1.to_xarray(), a2.to_xarray()]), - xr.merge([b1.to_xarray(), b2.to_xarray()]), - ] - ) + target = xr.merge([ + xr.merge([a1.to_xarray(), a2.to_xarray()]), + xr.merge([b1.to_xarray(), b2.to_xarray()]), + ]) return target, a1, a2, b1, b2 diff --git a/tests/netcdf/test_netcdf_fieldlist.py b/tests/netcdf/test_netcdf_fieldlist.py index 798b444df..e74c15c7e 100644 --- a/tests/netcdf/test_netcdf_fieldlist.py +++ b/tests/netcdf/test_netcdf_fieldlist.py @@ -43,7 +43,7 @@ def test_netcdf_fieldlist_string_coord(): def test_netcdf_fieldlist_bounds(): - """Check if having non string values in bounds does not cause a crash""" + """Check if having non string values in bounds does not cause a crash.""" import xarray as xr a = xr.DataArray( diff --git a/tests/netcdf/test_netcdf_geography.py b/tests/netcdf/test_netcdf_geography.py index 3c94a0c19..90845f828 100644 --- a/tests/netcdf/test_netcdf_geography.py +++ b/tests/netcdf/test_netcdf_geography.py @@ -12,12 +12,13 @@ import numpy as np import pytest -from earthkit.data import from_object -from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data import from_object, from_source +from earthkit.data.testing import ( + earthkit_examples_file, + earthkit_remote_examples_file, + earthkit_remote_test_data_file, + earthkit_test_data_file, +) from earthkit.data.utils import projections @@ -225,14 +226,12 @@ def test_netcdf_to_latlon_laea(): assert isinstance(v["lon"], np.ndarray) assert v["lon"].shape == (950, 1000) - ref = np.array( - [ - -35.034023999999995, - 73.93767587613708, - -8.229274420493763, - 41.13970495087975, - ] - ) + ref = np.array([ + -35.034023999999995, + 73.93767587613708, + -8.229274420493763, + 41.13970495087975, + ]) for i, x in enumerate(pos): assert np.isclose(v["lon"][x], ref[i]), f"{i=}, {x=}" @@ -240,14 +239,12 @@ def test_netcdf_to_latlon_laea(): assert isinstance(v["lat"], np.ndarray) assert v["lat"].shape == (950, 1000) - ref = np.array( - [ - 66.9821429989222, - 58.24673887576243, - 27.802844211251625, - 23.942342882929605, - ] - ) + ref = np.array([ + 66.9821429989222, + 58.24673887576243, + 27.802844211251625, + 23.942342882929605, + ]) for i, x in enumerate(pos): assert np.isclose(v["lat"][x], ref[i]), f"{i=}, {x=}" @@ -282,12 +279,10 @@ def test_netcdf_geography_2d_1(lat_name, lon_name): lats = [[50, 50, 50], [40, 40, 40], [30, 30, 30]] lons = [[0, 10, 20], [0, 10, 20], [0, 10, 20]] - data = np.array( - [ - [[11, 12, 13], [21, 22, 23], [31, 32, 33]], - [[14, 15, 16], [24, 25, 26], [34, 35, 36]], - ] - ) + data = np.array([ + [[11, 12, 13], [21, 22, 23], [31, 32, 33]], + [[14, 15, 16], [24, 25, 26], [34, 35, 36]], + ]) a = xr.Variable(dims, data) v = {"a": a} @@ -324,12 +319,10 @@ def test_netcdf_geography_2d_2(lat_name, lon_name): lon_name: (["y", "x"], np.array([[0, 10], [0, 10], [0, 10]])), } - data = np.array( - [ - [[11, 12], [21, 22], [31, 32]], - [[14, 15], [24, 25], [34, 35]], - ] - ) + data = np.array([ + [[11, 12], [21, 22], [31, 32]], + [[14, 15], [24, 25], [34, 35]], + ]) a = xr.Variable(dims, data) v = {"a": a} @@ -365,12 +358,10 @@ def test_netcdf_geography_2d_3(lat_name, lon_name): "level": np.array([700, 500]), } - data = np.array( - [ - [[11, 12], [21, 22], [31, 32]], - [[14, 15], [24, 25], [34, 35]], - ] - ) + data = np.array([ + [[11, 12], [21, 22], [31, 32]], + [[14, 15], [24, 25], [34, 35]], + ]) a = xr.Variable(dims, data) lat = xr.Variable({"y": 3, "x": 3}, np.array([[50, 50], [40, 40], [30, 30]])) @@ -409,12 +400,10 @@ def test_netcdf_geography_1d_1(lat_name, lon_name): lon_name: ("values", np.array([0, 10, 20, 0, 10, 20, 0, 10, 20])), } - data = np.array( - [ - [11, 12, 13, 21, 22, 23, 31, 32, 33], - [14, 15, 16, 24, 25, 26, 34, 35, 36], - ] - ) + data = np.array([ + [11, 12, 13, 21, 22, 23, 31, 32, 33], + [14, 15, 16, 24, 25, 26, 34, 35, 36], + ]) a = xr.Variable(dims, data) v = {"a": a} @@ -450,12 +439,10 @@ def test_netcdf_geography_1d_2(lat_name, lon_name): "level": np.array([700, 500]), } - data = np.array( - [ - [11, 12, 13, 21, 22, 23, 31, 32, 33], - [14, 15, 16, 24, 25, 26, 34, 35, 36], - ] - ) + data = np.array([ + [11, 12, 13, 21, 22, 23, 31, 32, 33], + [14, 15, 16, 24, 25, 26, 34, 35, 36], + ]) a = xr.Variable(dims, data) lat = xr.Variable({"values": 9}, np.array([50, 50, 50, 40, 40, 40, 30, 30, 30])) diff --git a/tests/netcdf/test_netcdf_iter.py b/tests/netcdf/test_netcdf_iter.py index e42abe39e..573aaa95d 100644 --- a/tests/netcdf/test_netcdf_iter.py +++ b/tests/netcdf/test_netcdf_iter.py @@ -13,8 +13,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file @pytest.mark.parametrize("group", ["param"]) diff --git a/tests/netcdf/test_netcdf_metadata.py b/tests/netcdf/test_netcdf_metadata.py index 81cd3aa0f..8ec28656a 100644 --- a/tests/netcdf/test_netcdf_metadata.py +++ b/tests/netcdf/test_netcdf_metadata.py @@ -14,8 +14,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source @pytest.mark.parametrize("mode", ["nc", "xr"]) diff --git a/tests/netcdf/test_netcdf_output.py b/tests/netcdf/test_netcdf_output.py index 70f1ed574..bde283aa2 100644 --- a/tests/netcdf/test_netcdf_output.py +++ b/tests/netcdf/test_netcdf_output.py @@ -15,10 +15,7 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import IN_GITHUB -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import write_to_file +from earthkit.data.testing import IN_GITHUB, WRITE_TO_FILE_METHODS, earthkit_examples_file, write_to_file @pytest.mark.parametrize("write_method", WRITE_TO_FILE_METHODS) diff --git a/tests/netcdf/test_netcdf_sel.py b/tests/netcdf/test_netcdf_sel.py index 39b570336..92567a942 100644 --- a/tests/netcdf/test_netcdf_sel.py +++ b/tests/netcdf/test_netcdf_sel.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source @pytest.mark.parametrize("mode", ["nc", "xr"]) diff --git a/tests/netcdf/test_netcdf_slice.py b/tests/netcdf/test_netcdf_slice.py index 042dc0dd6..b077ca054 100644 --- a/tests/netcdf/test_netcdf_slice.py +++ b/tests/netcdf/test_netcdf_slice.py @@ -13,8 +13,7 @@ import pytest from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source @pytest.mark.parametrize("mode", ["nc", "xr"]) diff --git a/tests/netcdf/test_netcdf_summary.py b/tests/netcdf/test_netcdf_summary.py index 1038d256f..567d8069f 100644 --- a/tests/netcdf/test_netcdf_summary.py +++ b/tests/netcdf/test_netcdf_summary.py @@ -10,8 +10,7 @@ import pytest -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source @pytest.mark.parametrize("mode", ["nc", "xr"]) diff --git a/tests/netcdf/test_netcdf_values.py b/tests/netcdf/test_netcdf_values.py index a391b7574..7944b91d0 100644 --- a/tests/netcdf/test_netcdf_values.py +++ b/tests/netcdf/test_netcdf_values.py @@ -12,8 +12,7 @@ import numpy as np import pytest -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import load_nc_or_xr_source +from earthkit.data.testing import earthkit_examples_file, load_nc_or_xr_source def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3): diff --git a/tests/normalize/test_normalize_aliases.py b/tests/normalize/test_normalize_aliases.py index 0a058323e..3364eabd2 100644 --- a/tests/normalize/test_normalize_aliases.py +++ b/tests/normalize/test_normalize_aliases.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data.decorators import alias_argument -from earthkit.data.decorators import normalize +from earthkit.data.decorators import alias_argument, normalize def func_x(x): diff --git a/tests/normalize/test_normalize_availability.py b/tests/normalize/test_normalize_availability.py index 104bd0ea5..a98fcf18b 100644 --- a/tests/normalize/test_normalize_availability.py +++ b/tests/normalize/test_normalize_availability.py @@ -16,8 +16,7 @@ import pytest import yaml -from earthkit.data.decorators import availability -from earthkit.data.decorators import normalize +from earthkit.data.decorators import availability, normalize from earthkit.data.utils.availability import Availability diff --git a/tests/normalize/test_normalize_date.py b/tests/normalize/test_normalize_date.py index 3e1a5c8b2..4cba9e461 100644 --- a/tests/normalize/test_normalize_date.py +++ b/tests/normalize/test_normalize_date.py @@ -222,12 +222,10 @@ def __init__(self, dates): def to_datetime_list(self): return self.dates - obj = CustomDateObject( - [ - datetime.datetime(2005, 8, 26, 18, 0), - datetime.datetime(2005, 8, 26, 18, 0), - ] - ) + obj = CustomDateObject([ + datetime.datetime(2005, 8, 26, 18, 0), + datetime.datetime(2005, 8, 26, 18, 0), + ]) assert date_formatted(obj) == "2020.05.13" diff --git a/tests/normalize/test_normalize_enum.py b/tests/normalize/test_normalize_enum.py index 46a31b45c..a4c5f43ba 100644 --- a/tests/normalize/test_normalize_enum.py +++ b/tests/normalize/test_normalize_enum.py @@ -12,8 +12,7 @@ import pytest from earthkit.data import ALL -from earthkit.data.arguments.earthkit_types import EnumListType -from earthkit.data.arguments.earthkit_types import EnumType +from earthkit.data.arguments.earthkit_types import EnumListType, EnumType from earthkit.data.decorators import normalize @@ -74,12 +73,10 @@ def test_enum_multiple(): def test_enum_multiple_2(): g = normalize("name", ["a", "b", "c"], multiple=True)(name_default_is_str_a) - assert g( - ( - "a", - "b", - ) - ) == ["a", "b"] + assert g(( + "a", + "b", + )) == ["a", "b"] def test_enum_multiple_ALL_1(): diff --git a/tests/normalize/test_normalize_errors.py b/tests/normalize/test_normalize_errors.py index 45b48a772..393cf8c20 100644 --- a/tests/normalize/test_normalize_errors.py +++ b/tests/normalize/test_normalize_errors.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data.decorators import availability -from earthkit.data.decorators import normalize +from earthkit.data.decorators import availability, normalize def name_no_default(name): diff --git a/tests/normalize/test_transformers.py b/tests/normalize/test_transformers.py index 1ec909ec8..aa8df2993 100644 --- a/tests/normalize/test_transformers.py +++ b/tests/normalize/test_transformers.py @@ -13,21 +13,22 @@ import pytest -from earthkit.data.arguments.earthkit_types import BoundingBoxType -from earthkit.data.arguments.earthkit_types import DateListType -from earthkit.data.arguments.earthkit_types import DateType -from earthkit.data.arguments.earthkit_types import EnumListType -from earthkit.data.arguments.earthkit_types import EnumType -from earthkit.data.arguments.earthkit_types import FloatListType -from earthkit.data.arguments.earthkit_types import FloatType -from earthkit.data.arguments.earthkit_types import IntListType -from earthkit.data.arguments.earthkit_types import IntType -from earthkit.data.arguments.earthkit_types import StrListType -from earthkit.data.arguments.earthkit_types import StrType -from earthkit.data.arguments.earthkit_types import VariableListType -from earthkit.data.arguments.earthkit_types import VariableType -from earthkit.data.arguments.transformers import FormatTransformer -from earthkit.data.arguments.transformers import TypeTransformer +from earthkit.data.arguments.earthkit_types import ( + BoundingBoxType, + DateListType, + DateType, + EnumListType, + EnumType, + FloatListType, + FloatType, + IntListType, + IntType, + StrListType, + StrType, + VariableListType, + VariableType, +) +from earthkit.data.arguments.transformers import FormatTransformer, TypeTransformer from earthkit.data.utils.bbox import BoundingBox enum = ("a", "b", "c") @@ -99,9 +100,10 @@ def test_formats(): assert FormatTransformer(None, type=DateType, format="%Y").transform(datetime.datetime(2000, 1, 1)) == "2000" - assert FormatTransformer(None, type=DateListType, format="%d").transform( - (datetime.datetime(2000, 1, 1), datetime.datetime(2000, 1, 2)) - ) == ["01", "02"] + assert FormatTransformer(None, type=DateListType, format="%d").transform(( + datetime.datetime(2000, 1, 1), + datetime.datetime(2000, 1, 2), + )) == ["01", "02"] with pytest.raises(Exception): # FIXME: Not sure what this should be assert FormatTransformer(None, type=VariableType, format="%4s").transform(42) == 0 diff --git a/tests/patterns/test_patterns.py b/tests/patterns/test_patterns.py index af340f21d..60d8f7eaa 100644 --- a/tests/patterns/test_patterns.py +++ b/tests/patterns/test_patterns.py @@ -13,9 +13,7 @@ import pytest -from earthkit.data.utils.patterns import Function -from earthkit.data.utils.patterns import Pattern -from earthkit.data.utils.patterns import Variable +from earthkit.data.utils.patterns import Function, Pattern, Variable @pytest.mark.parametrize( diff --git a/tests/readers/test_covjson_reader.py b/tests/readers/test_covjson_reader.py index b0536d0b6..cb9804e03 100644 --- a/tests/readers/test_covjson_reader.py +++ b/tests/readers/test_covjson_reader.py @@ -10,10 +10,8 @@ # import pytest -from earthkit.data import from_object -from earthkit.data import from_source -from earthkit.data.testing import NO_COVJSONKIT -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data import from_object, from_source +from earthkit.data.testing import NO_COVJSONKIT, earthkit_test_data_file def test_covjson(): diff --git a/tests/readers/test_geotiff_reader.py b/tests/readers/test_geotiff_reader.py index a7648b071..9ae6a6e0c 100644 --- a/tests/readers/test_geotiff_reader.py +++ b/tests/readers/test_geotiff_reader.py @@ -13,8 +13,7 @@ from earthkit.data import from_source from earthkit.data.readers.geotiff import GeoTIFFField -from earthkit.data.testing import NO_RIOXARRAY -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_RIOXARRAY, earthkit_test_data_file from earthkit.data.utils.projections import TransverseMercator diff --git a/tests/readers/test_grib_reader.py b/tests/readers/test_grib_reader.py index 575c6b9f9..86d153709 100644 --- a/tests/readers/test_grib_reader.py +++ b/tests/readers/test_grib_reader.py @@ -10,8 +10,7 @@ # from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_test_data_file def test_grib_len(): diff --git a/tests/readers/test_netcdf_reader.py b/tests/readers/test_netcdf_reader.py index fe7579919..8fc65fc55 100644 --- a/tests/readers/test_netcdf_reader.py +++ b/tests/readers/test_netcdf_reader.py @@ -19,12 +19,14 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_file from earthkit.data.readers.netcdf.field import NetCDFField -from earthkit.data.testing import IN_GITHUB -from earthkit.data.testing import NO_CDS -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file -from earthkit.data.testing import write_to_file +from earthkit.data.testing import ( + IN_GITHUB, + NO_CDS, + WRITE_TO_FILE_METHODS, + earthkit_examples_file, + earthkit_test_data_file, + write_to_file, +) def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3): @@ -204,8 +206,7 @@ def test_netcdf_multi_files(): @pytest.mark.no_eccodes @pytest.mark.skipif(IN_GITHUB, reason="Some runners crash in Xarray") def test_get_fields_missing_standard_name_attr_in_coord_array(): - """test _get_fields() can handle a missing 'standard_name' attr in coordinate data arrays""" - + """Test _get_fields() can handle a missing 'standard_name' attr in coordinate data arrays.""" # example dataset fs = from_source("file", earthkit_examples_file("test.nc")) ds = fs.to_xarray() diff --git a/tests/readers/test_pp_reader.py b/tests/readers/test_pp_reader.py index 71e2b301c..39041293e 100644 --- a/tests/readers/test_pp_reader.py +++ b/tests/readers/test_pp_reader.py @@ -13,9 +13,7 @@ from earthkit.data import from_source from earthkit.data.readers.netcdf.field import XArrayField -from earthkit.data.testing import NO_IRIS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_IRIS, earthkit_examples_file, earthkit_test_data_file @pytest.mark.skipif(NO_IRIS, reason="Iris or ncdata not installed") diff --git a/tests/readers/test_reader_padding_bytes.py b/tests/readers/test_reader_padding_bytes.py index 43aeb3d33..59dc19400 100644 --- a/tests/readers/test_reader_padding_bytes.py +++ b/tests/readers/test_reader_padding_bytes.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import config, from_source from earthkit.data.core.temporary import temp_file from earthkit.data.testing import earthkit_examples_file diff --git a/tests/sources/test_cds.py b/tests/sources/test_cds.py index 7483e02ea..88586821a 100644 --- a/tests/sources/test_cds.py +++ b/tests/sources/test_cds.py @@ -15,10 +15,7 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import NO_CDS -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import preserve_cwd -from earthkit.data.testing import write_to_file +from earthkit.data.testing import NO_CDS, WRITE_TO_FILE_METHODS, preserve_cwd, write_to_file CDS_TIMEOUT = pytest.CDS_TIMEOUT diff --git a/tests/sources/test_fdb.py b/tests/sources/test_fdb.py index 5efd6f3c4..434f43555 100644 --- a/tests/sources/test_fdb.py +++ b/tests/sources/test_fdb.py @@ -16,10 +16,7 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import NO_FDB -from earthkit.data.testing import NO_PROD_FDB -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_FDB, NO_PROD_FDB, earthkit_examples_file, earthkit_test_data_file @pytest.mark.long_test diff --git a/tests/sources/test_file.py b/tests/sources/test_file.py index 0a98fb9e0..47690c4b8 100644 --- a/tests/sources/test_file.py +++ b/tests/sources/test_file.py @@ -17,12 +17,14 @@ from earthkit.data import from_source from earthkit.data.core.temporary import temp_directory -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import make_tgz -from earthkit.data.testing import make_zip -from earthkit.data.testing import preserve_cwd -from earthkit.data.testing import write_to_file +from earthkit.data.testing import ( + WRITE_TO_FILE_METHODS, + earthkit_examples_file, + make_tgz, + make_zip, + preserve_cwd, + write_to_file, +) LOG = logging.getLogger(__name__) @@ -351,7 +353,6 @@ def test_file_netcdf_zip_with_single_file_2(): # print(ds_in) with temp_directory() as tmpdir: - path = os.path.join(tmpdir, "a.nc") ds_in["foo"].to_netcdf(path) diff --git a/tests/sources/test_gribjump.py b/tests/sources/test_gribjump.py index 3622ec8e7..84c444bd2 100644 --- a/tests/sources/test_gribjump.py +++ b/tests/sources/test_gribjump.py @@ -17,10 +17,8 @@ import yaml from earthkit.data import from_source -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_env -from earthkit.data.testing import NO_GRIBJUMP -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.core.temporary import temp_directory, temp_env +from earthkit.data.testing import NO_GRIBJUMP, earthkit_test_data_file @pytest.fixture @@ -102,28 +100,26 @@ def mask(): def arr_expected(): import numpy as np - arr_expected = np.array( + arr_expected = np.array([ [ - [ - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1607.31591797, - 1721.81591797, - ], - [ - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1702.31201172, - 1887.18701172, - ], - ] - ) + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1607.31591797, + 1721.81591797, + ], + [ + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1702.31201172, + 1887.18701172, + ], + ]) return arr_expected @@ -132,50 +128,44 @@ def ds_expected_with_coords(): import numpy as np import xarray as xr - arr_expected = np.array( - [ - [ - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1743.06591797, - 1607.31591797, - 1721.81591797, - ], - [ - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1641.43701172, - 1702.31201172, - 1887.18701172, - ], - ] - ) - latitude_expected = np.array( + arr_expected = np.array([ [ - 90.0, - 90.0, - 90.0, - 90.0, - 90.0, - 30.0, - 30.0, - ] - ) - longitude_expected = np.array( + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1743.06591797, + 1607.31591797, + 1721.81591797, + ], [ - 0.0, - 150.0, - 180.0, - 210.0, - 240.0, - 30.0, - 60.0, - ] - ) + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1641.43701172, + 1702.31201172, + 1887.18701172, + ], + ]) + latitude_expected = np.array([ + 90.0, + 90.0, + 90.0, + 90.0, + 90.0, + 30.0, + 30.0, + ]) + longitude_expected = np.array([ + 0.0, + 150.0, + 180.0, + 210.0, + 240.0, + 30.0, + 60.0, + ]) ds_expected = xr.Dataset( {"129": (("step", "index"), arr_expected)}, coords={ diff --git a/tests/sources/test_iris.py b/tests/sources/test_iris.py index d62fc1e5a..27cfe3d48 100644 --- a/tests/sources/test_iris.py +++ b/tests/sources/test_iris.py @@ -13,9 +13,7 @@ from earthkit.data import from_source from earthkit.data.readers.netcdf.field import XArrayField -from earthkit.data.testing import NO_IRIS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_IRIS, earthkit_examples_file, earthkit_test_data_file @pytest.mark.skip("Disabled at the moment") diff --git a/tests/sources/test_mars.py b/tests/sources/test_mars.py index 5eb52d4e0..346f97b54 100644 --- a/tests/sources/test_mars.py +++ b/tests/sources/test_mars.py @@ -13,14 +13,9 @@ import pytest -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import config, from_source from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import NO_MARS -from earthkit.data.testing import NO_MARS_API -from earthkit.data.testing import NO_MARS_DIRECT -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import write_to_file +from earthkit.data.testing import NO_MARS, NO_MARS_API, NO_MARS_DIRECT, WRITE_TO_FILE_METHODS, write_to_file YESTERDAY = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d") @@ -100,7 +95,6 @@ def test_mars_grib_multi(_args, req, _kwargs): @pytest.mark.skipif(NO_MARS, reason="No access to MARS") def test_mars_grib_parallel(): with config.temporary("number-of-download-threads", 4): - req = dict(param="t", levelist=[925, 850, 700, 500], date=YESTERDAY, split_on="levelist") ds = from_source( diff --git a/tests/sources/test_multi.py b/tests/sources/test_multi.py index 4939b4e12..fec59ac00 100644 --- a/tests/sources/test_multi.py +++ b/tests/sources/test_multi.py @@ -17,10 +17,8 @@ import pytest from earthkit.data import from_source -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import write_to_file +from earthkit.data.core.temporary import temp_directory, temp_file +from earthkit.data.testing import WRITE_TO_FILE_METHODS, write_to_file LOG = logging.getLogger(__name__) diff --git a/tests/sources/test_polytope.py b/tests/sources/test_polytope.py index 93d50788e..722a7e0e4 100644 --- a/tests/sources/test_polytope.py +++ b/tests/sources/test_polytope.py @@ -18,7 +18,7 @@ def test_no_polytope_client(monkeypatch): - "Check that a useful message is given in the absence of the polytope-client library" + """Check that a useful message is given in the absence of the polytope-client library.""" monkeypatch.setitem(sys.modules, "polytope", None) with pytest.raises(ImportError) as excinfo: from_source("polytope", None, None) diff --git a/tests/sources/test_url.py b/tests/sources/test_url.py index 7674f9c06..0754910ff 100644 --- a/tests/sources/test_url.py +++ b/tests/sources/test_url.py @@ -14,16 +14,16 @@ import pytest -from earthkit.data import config -from earthkit.data import from_source -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file -from earthkit.data.testing import WRITE_TO_FILE_METHODS -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import network_off -from earthkit.data.testing import write_to_file +from earthkit.data import config, from_source +from earthkit.data.core.temporary import temp_directory, temp_file +from earthkit.data.testing import ( + WRITE_TO_FILE_METHODS, + earthkit_examples_file, + earthkit_remote_examples_file, + earthkit_remote_test_data_file, + network_off, + write_to_file, +) @pytest.mark.skipif( # TODO: fix diff --git a/tests/sources/test_url_pattern.py b/tests/sources/test_url_pattern.py index 27df8b116..3943b8d34 100644 --- a/tests/sources/test_url_pattern.py +++ b/tests/sources/test_url_pattern.py @@ -12,8 +12,7 @@ import datetime from earthkit.data import from_source -from earthkit.data.testing import earthkit_remote_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file +from earthkit.data.testing import earthkit_remote_examples_file, earthkit_remote_test_data_file def test_url_pattern_source_1(): diff --git a/tests/sources/test_zarr.py b/tests/sources/test_zarr.py index 4348e972c..984ebec08 100644 --- a/tests/sources/test_zarr.py +++ b/tests/sources/test_zarr.py @@ -13,8 +13,7 @@ from earthkit.data import from_source from earthkit.data.readers.netcdf.field import XArrayField -from earthkit.data.testing import NO_ZARR -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_ZARR, earthkit_test_data_file @pytest.mark.skipif(NO_ZARR, reason="Zarr not installed") diff --git a/tests/targets/test_target_fdb.py b/tests/targets/test_target_fdb.py index 12a138fb9..83203e773 100644 --- a/tests/targets/test_target_fdb.py +++ b/tests/targets/test_target_fdb.py @@ -19,9 +19,7 @@ from earthkit.data.encoders.grib import GribEncoder from earthkit.data.targets import to_target from earthkit.data.targets.fdb import FDBTarget -from earthkit.data.testing import NO_FDB -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_FDB, earthkit_examples_file, earthkit_test_data_file TEST_GRIB_REQUEST = { "class": "od", diff --git a/tests/targets/test_target_file.py b/tests/targets/test_target_file.py index c7e803668..bd77a30ab 100644 --- a/tests/targets/test_target_file.py +++ b/tests/targets/test_target_file.py @@ -15,14 +15,11 @@ import pytest from earthkit.data import from_source -from earthkit.data.core.temporary import temp_directory -from earthkit.data.core.temporary import temp_file +from earthkit.data.core.temporary import temp_directory, temp_file from earthkit.data.encoders.grib import GribEncoder from earthkit.data.targets import to_target from earthkit.data.targets.file import FileTarget -from earthkit.data.testing import NO_RIOXARRAY -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import NO_RIOXARRAY, earthkit_examples_file, earthkit_test_data_file @pytest.mark.parametrize( diff --git a/tests/thread/test_thread_cached_property.py b/tests/thread/test_thread_cached_property.py index a11589ef9..fb48bc00e 100644 --- a/tests/thread/test_thread_cached_property.py +++ b/tests/thread/test_thread_cached_property.py @@ -23,14 +23,14 @@ def __init__(self, count=0): @thread_safe_cached_property def data(self): - """Property data""" + """Property data.""" time.sleep(1) self.count += 1 return self.count @thread_safe_cached_property def data_static(self): - """Property data_static""" + """Property data_static.""" time.sleep(1) self.count_static += 1 return self.count_static @@ -50,7 +50,7 @@ def __init__(self, count=0): @thread_safe_cached_property def _data(self): - """Property data""" + """Property data.""" time.sleep(1) self.count += 1 return _Wrapped(self.count) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index ede9f8126..26d31481a 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -17,10 +17,7 @@ import pandas as pd import xarray as xr -from earthkit.data import from_source -from earthkit.data import transform -from earthkit.data import translators -from earthkit.data import wrappers +from earthkit.data import from_source, transform, translators, wrappers from earthkit.data.testing import earthkit_test_data_file from earthkit.data.translators import ndarray as ndtranslator from earthkit.data.translators import pandas as pdtranslator diff --git a/tests/utils/test_dates.py b/tests/utils/test_dates.py index 56b88afb0..473457262 100644 --- a/tests/utils/test_dates.py +++ b/tests/utils/test_dates.py @@ -14,18 +14,20 @@ import numpy as np import pytest -from earthkit.data.utils.dates import date_to_grib -from earthkit.data.utils.dates import datetime_to_grib -from earthkit.data.utils.dates import mars_like_date_list -from earthkit.data.utils.dates import numpy_datetime_to_datetime -from earthkit.data.utils.dates import numpy_timedelta_to_timedelta -from earthkit.data.utils.dates import step_to_grib -from earthkit.data.utils.dates import time_to_grib -from earthkit.data.utils.dates import timedeltas_to_int -from earthkit.data.utils.dates import to_datetime -from earthkit.data.utils.dates import to_datetime_list -from earthkit.data.utils.dates import to_time -from earthkit.data.utils.dates import to_timedelta +from earthkit.data.utils.dates import ( + date_to_grib, + datetime_to_grib, + mars_like_date_list, + numpy_datetime_to_datetime, + numpy_timedelta_to_timedelta, + step_to_grib, + time_to_grib, + timedeltas_to_int, + to_datetime, + to_datetime_list, + to_time, + to_timedelta, +) # Change to utc once aware datetime objects will be used # tzinfo = datetime.timezone.utc diff --git a/tests/utils/test_download_examples.py b/tests/utils/test_download_examples.py index 2d8fc5774..caf12a21b 100644 --- a/tests/utils/test_download_examples.py +++ b/tests/utils/test_download_examples.py @@ -11,8 +11,7 @@ import os -from earthkit.data import download_example_file -from earthkit.data import remote_example_file +from earthkit.data import download_example_file, remote_example_file def test_download_example_file_single(tmpdir): diff --git a/tests/utils/test_module_inputs_wrapper.py b/tests/utils/test_module_inputs_wrapper.py index 4d9a2977b..b98a2b4ad 100644 --- a/tests/utils/test_module_inputs_wrapper.py +++ b/tests/utils/test_module_inputs_wrapper.py @@ -11,8 +11,7 @@ import pandas as pd import xarray as xr -from earthkit.data import from_object -from earthkit.data import from_source +from earthkit.data import from_object, from_source from earthkit.data.readers import Reader from earthkit.data.testing import earthkit_test_data_file from earthkit.data.utils import module_inputs_wrapper diff --git a/tests/utils/test_usermetadata.py b/tests/utils/test_usermetadata.py index ccb6f4c7c..2c7388712 100644 --- a/tests/utils/test_usermetadata.py +++ b/tests/utils/test_usermetadata.py @@ -19,14 +19,12 @@ def test_usermetadata_nogeom(): - meta = UserMetadata( - { - "shortName": "test", - "longName": "Test", - "date": 20180801, - "time": 300, - } - ) + meta = UserMetadata({ + "shortName": "test", + "longName": "Test", + "date": 20180801, + "time": 300, + }) assert meta["shortName"] == "test" assert meta["longName"] == "Test" @@ -113,12 +111,10 @@ def test_usermetadata_override(initial, update, expected): def test_usermetadata_override_shape(): meta = UserMetadata({}, shape=(10, 1)) - new_meta = meta.override( - { - "shortName": "2t", - "longName": "Temperature", - } - ) + new_meta = meta.override({ + "shortName": "2t", + "longName": "Temperature", + }) new_meta._shape = None assert new_meta._shape is None assert meta._shape is not None @@ -222,20 +218,18 @@ def test_usermetadata_forecast(data, ref_base, ref_valid, ref_step): def test_usermetadata_hdate_from_mars(): - meta = UserMetadata( - { - "class": "od", - "expver": "0001", - "stream": "enfh", - "type": "cf", - "levtype": "sfc", - "param": "167.128", - "date": "20180830", # Model version - "hdate": "20100830", # Start date of the forecasts - "time": "0000", # Forecast starts at 0am - "step": 12, # Forecast 12 hours ahead - } - ) + meta = UserMetadata({ + "class": "od", + "expver": "0001", + "stream": "enfh", + "type": "cf", + "levtype": "sfc", + "param": "167.128", + "date": "20180830", # Model version + "hdate": "20100830", # Start date of the forecasts + "time": "0000", # Forecast starts at 0am + "step": 12, # Forecast 12 hours ahead + }) ref_base_dt = datetime.datetime(2010, 8, 30, 0, 0) ref_valid_dt = datetime.datetime(2010, 8, 30, 12, 0) diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index f4d95b0c4..c51c6b6af 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data.utils import ensure_iterable -from earthkit.data.utils import ensure_sequence +from earthkit.data.utils import ensure_iterable, ensure_sequence @pytest.mark.parametrize( diff --git a/tests/wrappers/test_ndarray.py b/tests/wrappers/test_ndarray.py index a2a787adc..e87101ec2 100644 --- a/tests/wrappers/test_ndarray.py +++ b/tests/wrappers/test_ndarray.py @@ -11,8 +11,7 @@ import logging -from earthkit.data import from_object -from earthkit.data import wrappers +from earthkit.data import from_object, wrappers from earthkit.data.wrappers import ndarray as ndwrapper LOG = logging.getLogger(__name__) diff --git a/tests/wrappers/test_pandas.py b/tests/wrappers/test_pandas.py index 92fb4299a..a38279e05 100644 --- a/tests/wrappers/test_pandas.py +++ b/tests/wrappers/test_pandas.py @@ -17,8 +17,7 @@ import numpy as np import pandas as pd -from earthkit.data import from_object -from earthkit.data import wrappers +from earthkit.data import from_object, wrappers from earthkit.data.wrappers import pandas as pd_wrapper LOG = logging.getLogger(__name__) diff --git a/tests/wrappers/test_string.py b/tests/wrappers/test_string.py index 932358ec6..e7885bd11 100644 --- a/tests/wrappers/test_string.py +++ b/tests/wrappers/test_string.py @@ -12,8 +12,7 @@ import logging -from earthkit.data import from_object -from earthkit.data import wrappers +from earthkit.data import from_object, wrappers from earthkit.data.wrappers import string as strwrapper LOG = logging.getLogger(__name__) diff --git a/tests/wrappers/test_xarray.py b/tests/wrappers/test_xarray.py index d7466cd25..1f555c983 100644 --- a/tests/wrappers/test_xarray.py +++ b/tests/wrappers/test_xarray.py @@ -11,8 +11,7 @@ import pytest -from earthkit.data import from_object -from earthkit.data import wrappers +from earthkit.data import from_object, wrappers from earthkit.data.testing import earthkit_examples_file from earthkit.data.wrappers import xarray as xr_wrapper diff --git a/tests/xr_engine/conftest.py b/tests/xr_engine/conftest.py new file mode 100644 index 000000000..e669d95eb --- /dev/null +++ b/tests/xr_engine/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/tests/xr_engine/test_xr_attrs.py b/tests/xr_engine/test_xr_attrs.py index acef9f733..a805a93d5 100644 --- a/tests/xr_engine/test_xr_attrs.py +++ b/tests/xr_engine/test_xr_attrs.py @@ -9,21 +9,18 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pandas as pd import pytest +from xr_engine_fixtures import ( + compare_coords, + compare_dims, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import compare_dims # noqa: E402 - def _get_attrs(metadata): return {k: metadata.get(k, None) for k in ["gridType", "units"]} diff --git a/tests/xr_engine/test_xr_builder.py b/tests/xr_engine/test_xr_builder.py index 15986b3bb..301adcd56 100644 --- a/tests/xr_engine/test_xr_builder.py +++ b/tests/xr_engine/test_xr_builder.py @@ -9,9 +9,7 @@ # nor does it submit to any jurisdiction. # -import os import pickle -import sys import pytest @@ -19,9 +17,6 @@ from earthkit.data.core.temporary import temp_file from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) - # Testing internal structures in the xarray engine diff --git a/tests/xr_engine/test_xr_chunks.py b/tests/xr_engine/test_xr_chunks.py index 7e0855e24..5a6781010 100644 --- a/tests/xr_engine/test_xr_chunks.py +++ b/tests/xr_engine/test_xr_chunks.py @@ -13,8 +13,7 @@ import numpy as np import pytest -from earthkit.data import config -from earthkit.data import from_source +from earthkit.data import config, from_source from earthkit.data.testing import earthkit_remote_test_data_file diff --git a/tests/xr_engine/test_xr_dims.py b/tests/xr_engine/test_xr_dims.py index cdf062cf2..2c9a1f801 100644 --- a/tests/xr_engine/test_xr_dims.py +++ b/tests/xr_engine/test_xr_dims.py @@ -10,23 +10,20 @@ # import datetime -import os -import sys import pandas as pd import pytest +from xr_engine_fixtures import ( + compare_coords, + compare_dim_order, + compare_dims, + load_wrapped_fieldlist, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file from earthkit.data.utils.xarray.profile import Profile -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import compare_dim_order # noqa: E402 -from xr_engine_fixtures import compare_dims # noqa: E402 -from xr_engine_fixtures import load_wrapped_fieldlist # noqa: E402 - DS_LEV = { "class": ["od"], "param": ["t", "r"], @@ -135,7 +132,8 @@ def test_xr_dims_input_fieldlist(): ) def test_xr_dims_ds_lev(kwargs, var_key, variables, dim_keys): """Test for the internal profile/dimension object. Cannot use all the options since - many tasks are performed elsewhere in the engine.""" + many tasks are performed elsewhere in the engine. + """ # TODO: consider removing this test prof = Profile.make("mars", **kwargs) ds = load_wrapped_fieldlist(DS_LEV, prof) @@ -294,7 +292,8 @@ def test_xr_dims_ds_lev(kwargs, var_key, variables, dim_keys): ) def test_xr_dims_ds_date_lev(kwargs, var_key, variables, dims): """Test for the internal profile/dimension object. Cannot use all the options since - many tasks are performed elsewhere in the engine.""" + many tasks are performed elsewhere in the engine. + """ # TODO: consider removing this test prof = Profile.make("mars", **kwargs) @@ -334,7 +333,8 @@ def test_xr_dims_ds_date_lev(kwargs, var_key, variables, dims): ) def test_xr_dims_ds_sfc_and_pl(kwargs, var_key, variables, dim_keys): """Test for the internal profile/dimension object. Cannot use all the options since - many tasks are performed elsewhere in the engine.""" + many tasks are performed elsewhere in the engine. + """ # TODO: consider removing this test prof = Profile.make("mars", **kwargs) ds = load_wrapped_fieldlist(DS_DATE_SFC_PL, prof) diff --git a/tests/xr_engine/test_xr_engine.py b/tests/xr_engine/test_xr_engine.py index 8c1e5abe8..699c6765a 100644 --- a/tests/xr_engine/test_xr_engine.py +++ b/tests/xr_engine/test_xr_engine.py @@ -9,21 +9,18 @@ # nor does it submit to any jurisdiction. # -import os import pathlib -import sys import numpy as np import pytest +from xr_engine_fixtures import ( + compare_coords, + load_grib_data, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import load_grib_data # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("engine", ["earthkit", "cfgrib"]) @@ -214,34 +211,32 @@ def test_xr_engine_detailed_check_1(allow_holes, lazy_load, api): r = da[:, 0, :, 2, 9:12, :2] assert r.shape == (2, 2, 3, 2) - vals_ref = np.array( + vals_ref = np.array([ [ [ - [ - [269.00918579, 269.31680298], - [269.70254517, 269.81387329], - [267.50527954, 266.83828735], - ], - [ - [268.78610229, 268.80758667], - [269.52731323, 269.75680542], - [266.61813354, 267.12106323], - ], + [269.00918579, 269.31680298], + [269.70254517, 269.81387329], + [267.50527954, 266.83828735], ], [ - [ - [268.57771301, 269.03767395], - [269.33357239, 269.56111145], - [264.75154114, 266.55036926], - ], - [ - [268.08932495, 268.35983276], - [269.01803589, 269.02389526], - [264.29733276, 266.08248901], - ], + [268.78610229, 268.80758667], + [269.52731323, 269.75680542], + [266.61813354, 267.12106323], ], - ] - ) + ], + [ + [ + [268.57771301, 269.03767395], + [269.33357239, 269.56111145], + [264.75154114, 266.55036926], + ], + [ + [268.08932495, 268.35983276], + [269.01803589, 269.02389526], + [264.29733276, 266.08248901], + ], + ], + ]) assert np.allclose(r.values, vals_ref) r = da.loc[:, 0, :, 500, 0, 0] @@ -408,34 +403,32 @@ def test_xr_engine_detailed_check_2(allow_holes, lazy_load, api): r = da[:, 0, :, 2, 9:12, :2] assert r.shape == (2, 2, 3, 2) - vals_ref = np.array( + vals_ref = np.array([ [ [ - [ - [269.00918579, 269.31680298], - [269.70254517, 269.81387329], - [267.50527954, 266.83828735], - ], - [ - [268.78610229, 268.80758667], - [269.52731323, 269.75680542], - [266.61813354, 267.12106323], - ], + [269.00918579, 269.31680298], + [269.70254517, 269.81387329], + [267.50527954, 266.83828735], ], [ - [ - [268.57771301, 269.03767395], - [269.33357239, 269.56111145], - [264.75154114, 266.55036926], - ], - [ - [268.08932495, 268.35983276], - [269.01803589, 269.02389526], - [264.29733276, 266.08248901], - ], + [268.78610229, 268.80758667], + [269.52731323, 269.75680542], + [266.61813354, 267.12106323], ], - ] - ) + ], + [ + [ + [268.57771301, 269.03767395], + [269.33357239, 269.56111145], + [264.75154114, 266.55036926], + ], + [ + [268.08932495, 268.35983276], + [269.01803589, 269.02389526], + [264.29733276, 266.08248901], + ], + ], + ]) assert np.allclose(r.values, vals_ref) r = da.loc[:, 0, :, 500, 0, 0] @@ -602,18 +595,16 @@ def test_xr_engine_detailed_flatten_check_1(allow_holes, stream, lazy_load, rele r = da[:, 0, :, 2, [9 * 36, 10 * 36, 11 * 36]] assert r.shape == (2, 2, 3) - vals_ref = np.array( + vals_ref = np.array([ [ - [ - [269.00918579, 269.70254517, 267.50527954], - [268.78610229, 269.52731323, 266.61813354], - ], - [ - [268.57771301, 269.33357239, 264.75154114], - [268.08932495, 269.01803589, 264.29733276], - ], - ] - ) + [269.00918579, 269.70254517, 267.50527954], + [268.78610229, 269.52731323, 266.61813354], + ], + [ + [268.57771301, 269.33357239, 264.75154114], + [268.08932495, 269.01803589, 264.29733276], + ], + ]) v_ek = ds_ek_ref.sel(param="t", time=0, levelist=500).to_numpy(flatten=True) assert np.allclose(r.values.flatten(), v_ek[:, [9 * 36, 10 * 36, 11 * 36]].flatten()) @@ -778,18 +769,16 @@ def test_xr_engine_detailed_flatten_check_2(allow_holes, stream, lazy_load, rele r = da[:, 0, :, 2, [9 * 36, 10 * 36, 11 * 36]] assert r.shape == (2, 2, 3) - vals_ref = np.array( + vals_ref = np.array([ [ - [ - [269.00918579, 269.70254517, 267.50527954], - [268.78610229, 269.52731323, 266.61813354], - ], - [ - [268.57771301, 269.33357239, 264.75154114], - [268.08932495, 269.01803589, 264.29733276], - ], - ] - ) + [269.00918579, 269.70254517, 267.50527954], + [268.78610229, 269.52731323, 266.61813354], + ], + [ + [268.57771301, 269.33357239, 264.75154114], + [268.08932495, 269.01803589, 264.29733276], + ], + ]) v_ek = ds_ek_ref.sel(param="t", time=0, levelist=500).to_numpy(flatten=True) assert np.allclose(r.values.flatten(), v_ek[:, [9 * 36, 10 * 36, 11 * 36]].flatten()) diff --git a/tests/xr_engine/test_xr_ens.py b/tests/xr_engine/test_xr_ens.py index 331334b68..fe465068f 100644 --- a/tests/xr_engine/test_xr_ens.py +++ b/tests/xr_engine/test_xr_ens.py @@ -9,18 +9,13 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from xr_engine_fixtures import compare_dims from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("allow_holes", [False, True]) diff --git a/tests/xr_engine/test_xr_grid.py b/tests/xr_engine/test_xr_grid.py index d3f292d7d..5d1f35f51 100644 --- a/tests/xr_engine/test_xr_grid.py +++ b/tests/xr_engine/test_xr_grid.py @@ -16,8 +16,7 @@ import yaml from earthkit.data import from_source -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_remote_test_data_file, earthkit_test_data_file from earthkit.data.utils import ensure_iterable diff --git a/tests/xr_engine/test_xr_level.py b/tests/xr_engine/test_xr_level.py index 9e5bb8d51..fe3d5fc06 100644 --- a/tests/xr_engine/test_xr_level.py +++ b/tests/xr_engine/test_xr_level.py @@ -9,19 +9,14 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from xr_engine_fixtures import compare_dims from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file from earthkit.data.utils.xarray.profile import PROFILE_CONF -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("allow_holes", [False, True]) diff --git a/tests/xr_engine/test_xr_lod.py b/tests/xr_engine/test_xr_lod.py index 3ae566448..4eb040184 100644 --- a/tests/xr_engine/test_xr_lod.py +++ b/tests/xr_engine/test_xr_lod.py @@ -9,18 +9,13 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from xr_engine_fixtures import compare_dims from earthkit.data import from_source -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.fixture def xr_lod_latlon(): @@ -172,12 +167,10 @@ def test_xr_engine_lod_nongeo(allow_holes, lazy_load, xr_lod_nongeo): assert ds["t"].shape == (2, 6) assert ds["u"].shape == (2, 6) - ref = np.array( - [ - [1, 2, 3, 4, 5, 6], - [1, 2, 3, 4, 5, 6], - ] - ) + ref = np.array([ + [1, 2, 3, 4, 5, 6], + [1, 2, 3, 4, 5, 6], + ]) assert np.allclose(ds["t"].values, ref) assert np.allclose(ds["u"].values, ref) diff --git a/tests/xr_engine/test_xr_memory.py b/tests/xr_engine/test_xr_memory.py index a5099d9bb..63a91d811 100644 --- a/tests/xr_engine/test_xr_memory.py +++ b/tests/xr_engine/test_xr_memory.py @@ -9,15 +9,10 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest - -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import load_grib_data # noqa: E402 +from xr_engine_fixtures import load_grib_data class Counter: diff --git a/tests/xr_engine/test_xr_numpy.py b/tests/xr_engine/test_xr_numpy.py index dd2603135..f6cf1e8ef 100644 --- a/tests/xr_engine/test_xr_numpy.py +++ b/tests/xr_engine/test_xr_numpy.py @@ -12,11 +12,10 @@ import numpy as np import pytest +from earthkit.utils.array import array_namespace as eku_array_namespace from earthkit.data import from_source -from earthkit.data.testing import check_array_type -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.utils.array import array_namespace as eku_array_namespace +from earthkit.data.testing import check_array_type, earthkit_remote_test_data_file _NUMPY = eku_array_namespace("numpy") diff --git a/tests/xr_engine/test_xr_remapping.py b/tests/xr_engine/test_xr_remapping.py index 1751d94a7..bc5c2b942 100644 --- a/tests/xr_engine/test_xr_remapping.py +++ b/tests/xr_engine/test_xr_remapping.py @@ -9,19 +9,16 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from xr_engine_fixtures import ( + compare_coords, + compare_dims, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("allow_holes", [False, True]) diff --git a/tests/xr_engine/test_xr_time.py b/tests/xr_engine/test_xr_time.py index 6d9df5346..6792a83c6 100644 --- a/tests/xr_engine/test_xr_time.py +++ b/tests/xr_engine/test_xr_time.py @@ -9,20 +9,17 @@ # nor does it submit to any jurisdiction. # -import os -import sys import numpy as np import pytest +from xr_engine_fixtures import ( + compare_coords, + compare_dims, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("allow_holes", [False, True]) @@ -166,9 +163,9 @@ def test_xr_time_basic(allow_holes, kwargs, dims, step_units): compare_dims(ds, dims, order_ref_var="t") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -252,9 +249,9 @@ def test_xr_time_seasonal_monthly_indexing_date(allow_holes, kwargs, dims, step_ compare_dims(ds, dims, order_ref_var="2t") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -397,9 +394,9 @@ def test_xr_time_seasonal_monthly_simple(allow_holes, kwargs, dims, step_units): compare_dims(ds, dims, order_ref_var="2t") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -520,9 +517,9 @@ def test_xr_valid_time_coord(allow_holes, kwargs, dims, step_units, coords): compare_coords(ds, coords) if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -559,9 +556,9 @@ def test_xr_time_step_range_1(allow_holes, lazy_load, kwargs, dims, step_units): compare_dims(ds, dims, order_ref_var="10fg6") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -605,9 +602,9 @@ def test_xr_time_step_range_2(allow_holes, lazy_load, kwargs, dims, step_units): compare_dims(ds, dims, order_ref_var="lsp") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) @pytest.mark.cache @@ -683,6 +680,6 @@ def test_xr_time_step_seconds(allow_holes, lazy_load, kwargs, dims, step_units): compare_dims(ds, dims, order_ref_var="t") if step_units is not None: - assert ( - ds[step_units[0]].attrs["units"] == step_units[1] - ), f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + assert ds[step_units[0]].attrs["units"] == step_units[1], ( + f"step units mismatch {ds[step_units[0]].attrs['units']} != {step_units[1]}" + ) diff --git a/tests/xr_engine/test_xr_torch.py b/tests/xr_engine/test_xr_torch.py index c86b2a698..4bb682c35 100644 --- a/tests/xr_engine/test_xr_torch.py +++ b/tests/xr_engine/test_xr_torch.py @@ -12,14 +12,13 @@ import pytest -from earthkit.data import from_source -from earthkit.data.testing import check_array_type -from earthkit.data.testing import earthkit_remote_test_data_file - # from earthkit.utils.array import _TORCH from earthkit.utils.array import array_namespace as eku_array_namespace from earthkit.utils.array.testing.testing import NO_TORCH +from earthkit.data import from_source +from earthkit.data.testing import check_array_type, earthkit_remote_test_data_file + _TORCH = None if not NO_TORCH: _TORCH = eku_array_namespace("torch") diff --git a/tests/xr_engine/test_xr_variable.py b/tests/xr_engine/test_xr_variable.py index eb6cb3a73..9f6ffce1a 100644 --- a/tests/xr_engine/test_xr_variable.py +++ b/tests/xr_engine/test_xr_variable.py @@ -9,19 +9,16 @@ # nor does it submit to any jurisdiction. # -import os -import sys import pytest +from xr_engine_fixtures import ( + compare_coords, + compare_dims, +) from earthkit.data import from_source from earthkit.data.testing import earthkit_remote_test_data_file -here = os.path.dirname(__file__) -sys.path.insert(0, here) -from xr_engine_fixtures import compare_coords # noqa: E402 -from xr_engine_fixtures import compare_dims # noqa: E402 - @pytest.mark.cache @pytest.mark.parametrize("allow_holes", [False, True]) diff --git a/tests/xr_engine/test_xr_write.py b/tests/xr_engine/test_xr_write.py index 838f70bc8..c94dfc24e 100644 --- a/tests/xr_engine/test_xr_write.py +++ b/tests/xr_engine/test_xr_write.py @@ -12,8 +12,7 @@ import numpy as np import pytest -from earthkit.data import from_source -from earthkit.data import to_target +from earthkit.data import from_source, to_target from earthkit.data.core.temporary import temp_file from earthkit.data.testing import earthkit_remote_test_data_file from earthkit.data.utils.dates import datetime_to_grib diff --git a/tests/xr_engine/xr_engine_fixtures.py b/tests/xr_engine/xr_engine_fixtures.py index 61c385a8a..bdfe09227 100644 --- a/tests/xr_engine/xr_engine_fixtures.py +++ b/tests/xr_engine/xr_engine_fixtures.py @@ -12,9 +12,7 @@ import numpy as np from earthkit.data import from_source -from earthkit.data.testing import earthkit_examples_file -from earthkit.data.testing import earthkit_remote_test_data_file -from earthkit.data.testing import earthkit_test_data_file +from earthkit.data.testing import earthkit_examples_file, earthkit_remote_test_data_file, earthkit_test_data_file from earthkit.data.utils.xarray.fieldlist import XArrayInputFieldList