diff --git a/.gitignore b/.gitignore index 5d249e245..ebda864a7 100644 --- a/.gitignore +++ b/.gitignore @@ -213,7 +213,7 @@ docs/examples/*.tar docs/examples/*.zip docs/examples/_* docs/examples/earthkit_use_cases/*.grib -docs/examples/_fdb +docs/examples/*/_* docs/experimental/_* docs/experimental/*.grib* docs/experimental/*.pp* diff --git a/docs/examples/bufr_synop.ipynb b/docs/examples/bufr/bufr_synop.ipynb similarity index 97% rename from docs/examples/bufr_synop.ipynb rename to docs/examples/bufr/bufr_synop.ipynb index 860c99486..df957be5f 100644 --- a/docs/examples/bufr_synop.ipynb +++ b/docs/examples/bufr/bufr_synop.ipynb @@ -39,14 +39,39 @@ "execution_count": 2, "id": "f0430464-fd40-4964-b64d-4629ddd98909", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "

BUFR file
Path: synop_10.bufr size: 2.1 KiB
Available types: pandas, featurelist

" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ds = ekd.from_source(\"file\", \"synop_10.bufr\").to_featurelist()" + "ds = ekd.from_source(\"file\", \"synop_10.bufr\")\n", + "ds" ] }, { "cell_type": "code", "execution_count": 3, + "id": "5eda4c70-0eea-4cc2-ba84-a915d7048eec", + "metadata": {}, + "outputs": [], + "source": [ + "ds = ds.to_featurelist()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "id": "07571864-48af-451b-be4a-6a42436d193c", "metadata": {}, "outputs": [ @@ -56,7 +81,7 @@ "10" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -82,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "30e7749a-ea8b-43d2-90d2-31907f228513", "metadata": { "editable": true, @@ -319,7 +344,7 @@ "9 91701 -2.77 -171.72 " ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -355,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "53ac87d6-09b9-46c3-bc54-07fd2a175b05", "metadata": { "editable": true, @@ -481,7 +506,7 @@ "9 -2.77 -171.72 2.0 302.1" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -510,7 +535,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "26cebaca-9b17-4e15-9564-d92804886ea9", "metadata": {}, "outputs": [ @@ -569,7 +594,7 @@ "1 51.35 112.47 743.0 287.4 30846" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -591,7 +616,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "04ec7393-8969-4f46-82a6-5feb8f8a54c8", "metadata": {}, "outputs": [ @@ -655,7 +680,7 @@ "2 -26.50 29.98 1774.0 281.9" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -677,7 +702,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "d75fe63b-baaa-43fa-8689-c3838e6b0fe5", "metadata": {}, "outputs": [ @@ -749,7 +774,7 @@ "3 -29.60 31.12 105 299.8" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -771,7 +796,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "c7611278-9426-46fd-8c49-402545122414", "metadata": {}, "outputs": [ @@ -835,7 +860,7 @@ "2 -2.77 -171.72 2 302.1" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } diff --git a/docs/examples/bufr_temp.ipynb b/docs/examples/bufr/bufr_temp.ipynb similarity index 100% rename from docs/examples/bufr_temp.ipynb rename to docs/examples/bufr/bufr_temp.ipynb diff --git a/docs/examples/bufr/index.rst b/docs/examples/bufr/index.rst new file mode 100644 index 000000000..c6a771405 --- /dev/null +++ b/docs/examples/bufr/index.rst @@ -0,0 +1,10 @@ +.. _bufr_examples: + +BUFR +++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + bufr* diff --git a/docs/examples/bufr/synop_10.bufr b/docs/examples/bufr/synop_10.bufr new file mode 120000 index 000000000..4383aebdf --- /dev/null +++ b/docs/examples/bufr/synop_10.bufr @@ -0,0 +1 @@ +../synop_10.bufr \ No newline at end of file diff --git a/docs/examples/bufr/temp_10.bufr b/docs/examples/bufr/temp_10.bufr new file mode 120000 index 000000000..664b8e30a --- /dev/null +++ b/docs/examples/bufr/temp_10.bufr @@ -0,0 +1 @@ +../temp_10.bufr \ No newline at end of file diff --git a/docs/examples/covjson/index.rst b/docs/examples/covjson/index.rst new file mode 100644 index 000000000..dc4c6a0d7 --- /dev/null +++ b/docs/examples/covjson/index.rst @@ -0,0 +1,13 @@ +.. _covjson_examples: + + +CoverageJSON ++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + polytope_time_series.ipynb + polytope_polygon_coverage.ipynb + polytope_vertical_profile.ipynb diff --git a/docs/examples/polytope_polygon_coverage.ipynb b/docs/examples/covjson/polytope_polygon_coverage.ipynb similarity index 100% rename from docs/examples/polytope_polygon_coverage.ipynb rename to docs/examples/covjson/polytope_polygon_coverage.ipynb diff --git a/docs/examples/polytope_time_series.ipynb b/docs/examples/covjson/polytope_time_series.ipynb similarity index 99% rename from docs/examples/polytope_time_series.ipynb rename to docs/examples/covjson/polytope_time_series.ipynb index 574c0d5d0..2161243ed 100644 --- a/docs/examples/polytope_time_series.ipynb +++ b/docs/examples/covjson/polytope_time_series.ipynb @@ -776,7 +776,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/polytope_vertical_profile.ipynb b/docs/examples/covjson/polytope_vertical_profile.ipynb similarity index 100% rename from docs/examples/polytope_vertical_profile.ipynb rename to docs/examples/covjson/polytope_vertical_profile.ipynb diff --git a/docs/examples/dict/fields_from_dict_in_loop.ipynb b/docs/examples/dict/fields_from_dict_in_loop.ipynb new file mode 100644 index 000000000..c03010289 --- /dev/null +++ b/docs/examples/dict/fields_from_dict_in_loop.ipynb @@ -0,0 +1,878 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "69237222-3be8-4f6b-985c-0a8181730dfb", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Build fieldlist from dicts in a loop" + ] + }, + { + "cell_type": "markdown", + "id": "0e02de29-ccd2-4f2c-85c5-b5218179f56f", + "metadata": {}, + "source": [ + "#### Distinct latitudes/longitudes" + ] + }, + { + "cell_type": "markdown", + "id": "dd0ba727-0fcc-4c4d-a288-73f01da2921e", + "metadata": {}, + "source": [ + "In this example each field is defined by a dict using generic metadata keys. The latitudes and longitudes specified as disctinct values, the 2D field geometry will be automatically built from them." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a17559f9-cba1-4dd4-bc30-14a9be9891ae", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "prototype = {\n", + " \"geography\": {\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0]\n", + " },\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"time\": {\"valid_datetime\": \"2018-08-01T09:00:00Z\"},\n", + " }\n", + "\n", + "d = [\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"d\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"d\"}, \"vertical\": {\"level\": 600}, **prototype},\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "2e6a5236-09d4-4b61-be4a-e4d860c52d41", + "metadata": {}, + "source": [ + "Create a fieldlist from the data above in a loop." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1f902ec4-1eac-4f81-8fff-926a5640363c", + "metadata": {}, + "outputs": [], + "source": [ + "from earthkit.data import create_fieldlist\n", + "from earthkit.data import Field\n", + "\n", + "ds = create_fieldlist([Field.from_dict(f) for f in d])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "54099091-f8c6-49d0-997e-eb2c69137dfc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days500unknownNone_distinct_ll
1t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
2u2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days500unknownNone_distinct_ll
3u2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
4d2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
5d2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days600unknownNone_distinct_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "2 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "3 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "4 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "5 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 850 unknown None \n", + "2 0 days 500 unknown None \n", + "3 0 days 850 unknown None \n", + "4 0 days 850 unknown None \n", + "5 0 days 600 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll \n", + "2 _distinct_ll \n", + "3 _distinct_ll \n", + "4 _distinct_ll \n", + "5 _distinct_ll " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "df8d7402-3f11-4eab-9a85-fbc8b6e9955f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Field
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
number_of_values6
array_typendarray
array_dtypeint64
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
variablet
unitsdimensionless
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
valid_datetime2018-08-01 09:00:00+00:00
base_datetime2018-08-01 09:00:00+00:00
step0:00:00
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
level500
layerNone
level_typeunknown
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
memberNone
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
grid_specNone
grid_type_distinct_ll
shape(3, 2)
area(10.0, 20.0, -10.0, 40.0)
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + "Field(t, 2018-08-01 09:00:00+00:00, 2018-08-01 09:00:00+00:00, 0:00:00, 500, unknown, None, _distinct_ll)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1b841c36-1f50-4851-b69f-53e5e0e1f130", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([[ 10., 10.],\n", + " [ 0., 0.],\n", + " [-10., -10.]]),\n", + " array([[20., 40.],\n", + " [20., 40.],\n", + " [20., 40.]]))" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0].geography.latlons()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e771527b-0b8f-4ac2-b446-ab539614b48b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[[1, 2],\n", + " [3, 4],\n", + " [5, 6]],\n", + "\n", + " [[1, 2],\n", + " [3, 4],\n", + " [5, 6]],\n", + "\n", + " [[1, 2],\n", + " [3, 4],\n", + " [5, 6]],\n", + "\n", + " [[1, 2],\n", + " [3, 4],\n", + " [5, 6]],\n", + "\n", + " [[1, 2],\n", + " [3, 4],\n", + " [5, 6]],\n", + "\n", + " [[1, 2],\n", + " [3, 4],\n", + " [5, 6]]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.to_numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "3f11ca68-d566-44d9-9ac2-5c33a5a0abab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[500, 850, 500, 850, 850, 600]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.get(\"vertical.level\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e85b74cc-7524-49ea-9c94-9cdbec2e80dc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "datetime.datetime(2018, 8, 1, 9, 0, tzinfo=datetime.timezone.utc)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0].time.valid_datetime()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "526a4533-a274-4549-bc08-441ae19ecb5a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days500unknownNone_distinct_ll
1t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 850 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.sel({\"parameter.variable\": \"t\"}).ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1b69116f-7852-4ef9-85b6-51d1c4227da2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days500unknownNone_distinct_ll
1u2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days500unknownNone_distinct_ll
2d2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days600unknownNone_distinct_ll
3t2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
4u2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
5d2018-08-01 09:00:00+00:002018-08-01 09:00:00+00:000 days850unknownNone_distinct_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "2 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "3 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "4 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "5 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 500 unknown None \n", + "2 0 days 600 unknown None \n", + "3 0 days 850 unknown None \n", + "4 0 days 850 unknown None \n", + "5 0 days 850 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll \n", + "2 _distinct_ll \n", + "3 _distinct_ll \n", + "4 _distinct_ll \n", + "5 _distinct_ll " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.order_by(\"vertical.level\").ls()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d971846b-4834-4b06-98f3-bacb3925891d", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/dict/index.rst b/docs/examples/dict/index.rst new file mode 100644 index 000000000..318b42cb4 --- /dev/null +++ b/docs/examples/dict/index.rst @@ -0,0 +1,13 @@ +.. _dict_examples: + +Dictionary input +++++++++++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + fields_from_dict_in_loop.ipynb + list_of_dicts_overview + list_of_dicts_geography + list_of_dicts_to_xarray diff --git a/docs/examples/list_of_dicts_geography.ipynb b/docs/examples/dict/list_of_dicts_geography.ipynb similarity index 70% rename from docs/examples/list_of_dicts_geography.ipynb rename to docs/examples/dict/list_of_dicts_geography.ipynb index 371efa101..10d946804 100644 --- a/docs/examples/list_of_dicts_geography.ipynb +++ b/docs/examples/dict/list_of_dicts_geography.ipynb @@ -63,13 +63,13 @@ "def build_lod(data):\n", " prototype = {\n", " **data,\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " \"param\": \"t\", \n", + " \"time\": {\"valid_datetime\": \"2018-08-01T09:00:00Z\"},\n", + " \"parameter\": {\"variable\": \"t\"}, \n", " }\n", " \n", " lod = [\n", - " {\"level\": 500, **prototype},\n", - " {\"level\": 850, **prototype},\n", + " {\"vertical\": {\"level\": 500}, **prototype},\n", + " {\"vertical\": {\"level\": 850}, **prototype},\n", " ]\n", " return lod" ] @@ -135,41 +135,55 @@ " \n", " \n", " \n", - " param\n", - " level\n", - " base_datetime\n", - " valid_datetime\n", - " step\n", - " number\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 1\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", "\n", "" ], "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 t 850 None 2018-08-01T09:00:00+00:00 None None" + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 850 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll " ] }, "execution_count": 2, @@ -178,11 +192,13 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0]),\n", + "data = {\"geography\": {\n", + " \"latitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0])\n", + " },\n", " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", "ds.ls()" ] }, @@ -202,12 +218,12 @@ "data": { "text/plain": [ "((3, 2),\n", - " {'lat': array([[ 10., 10.],\n", + " (array([[ 10., 10.],\n", " [ 0., 0.],\n", " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", + " array([[20., 40.],\n", " [20., 40.],\n", - " [20., 40.]])},\n", + " [20., 40.]])),\n", " array([[1, 2],\n", " [3, 4],\n", " [5, 6]]))" @@ -219,7 +235,7 @@ } ], "source": [ - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -252,12 +268,12 @@ "data": { "text/plain": [ "((3, 2),\n", - " {'lat': array([[ 10., 10.],\n", + " (array([[ 10., 10.],\n", " [ 0., 0.],\n", " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", + " array([[20., 40.],\n", " [20., 40.],\n", - " [20., 40.]])},\n", + " [20., 40.]])),\n", " array([[1, 2],\n", " [3, 4],\n", " [5, 6]]))" @@ -269,12 +285,14 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0]),\n", + "data = {\"geography\": {\n", + " \"latitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0])\n", + " },\n", " \"values\": np.array([[1, 2], [3, 4], [5, 6]])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -306,16 +324,7 @@ { "data": { "text/plain": [ - "((3, 2),\n", - " {'lat': array([[ 10., 10.],\n", - " [ 0., 0.],\n", - " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", - " [20., 40.],\n", - " [20., 40.]])},\n", - " array([[1, 2],\n", - " [3, 4],\n", - " [5, 6]]))" + "((6,), (None, None), array([1, 2, 3, 4, 5, 6]))" ] }, "execution_count": 5, @@ -324,12 +333,14 @@ } ], "source": [ - "data = {\"distinctLatitudes\": np.array([10.0, 0.0, -10.0]),\n", - " \"distinctLongitudes\": np.array([20, 40.0]),\n", + "data = {\"geography\": {\n", + " \"distinctLatitudes\": np.array([10.0, 0.0, -10.0]),\n", + " \"distinctLongitudes\": np.array([20, 40.0])\n", + " },\n", " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -391,8 +402,8 @@ "data": { "text/plain": [ "((6,),\n", - " {'lat': array([ 10., 10., 0., 0., -10., -10.]),\n", - " 'lon': array([20., 40., 20., 40., 20., 40.])},\n", + " (array([ 10., 10., 0., 0., -10., -10.]),\n", + " array([20., 40., 20., 40., 20., 40.])),\n", " array([1, 2, 3, 4, 5, 6]))" ] }, @@ -402,12 +413,14 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10., 10., 0., 0., -10.0, -10.0]),\n", - " \"longitudes\": np.array([20, 40.0, 20., 40., 20., 40.]),\n", + "data = {\"geography\": {\n", + " \"latitudes\": np.array([10., 10., 0., 0., -10.0, -10.0]),\n", + " \"longitudes\": np.array([20, 40.0, 20., 40., 20., 40.])\n", + " },\n", " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -440,12 +453,12 @@ "data": { "text/plain": [ "((3, 2),\n", - " {'lat': array([[ 10., 10.],\n", + " (array([[ 10., 10.],\n", " [ 0., 0.],\n", " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", + " array([[20., 40.],\n", " [20., 40.],\n", - " [20., 40.]])},\n", + " [20., 40.]])),\n", " array([[1, 2],\n", " [3, 4],\n", " [5, 6]]))" @@ -457,12 +470,14 @@ } ], "source": [ - "data = {\"latitudes\": np.array([[10., 10.], [0., 0.], [-10.0, -10.0]]),\n", - " \"longitudes\": np.array([[20.0, 40.], [20., 40.], [20., 40.]]),\n", + "data = {\"geography\": {\n", + " \"latitudes\": np.array([[10., 10.], [0., 0.], [-10.0, -10.0]]),\n", + " \"longitudes\": np.array([[20.0, 40.], [20., 40.], [20., 40.]])\n", + " },\n", " \"values\": np.array([1, 2, 3, 4, 5, 6])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -495,8 +510,7 @@ "data": { "text/plain": [ "((5,),\n", - " {'lat': array([10., 10., 6., 6., 6.]),\n", - " 'lon': array([ 20., 40., -40., 0., 40.])},\n", + " (array([10., 10., 6., 6., 6.]), array([ 20., 40., -40., 0., 40.])),\n", " array([1, 2, 3, 4, 5]))" ] }, @@ -506,12 +520,14 @@ } ], "source": [ - "data = {\"latitudes\": np.array([10., 10., 6., 6., 6.]),\n", - " \"longitudes\": np.array([20, 40.0, -40., 0., 40.]),\n", + "data = {\"geography\": {\n", + " \"latitudes\": np.array([10., 10., 6., 6., 6.]),\n", + " \"longitudes\": np.array([20, 40.0, -40., 0., 40.])\n", + " },\n", " \"values\": np.array([1, 2, 3, 4, 5])}\n", "\n", - "ds = ekd.from_source(\"list-of-dicts\", build_lod(data))\n", - "ds[0].shape, ds[0].to_latlon(), ds[0].to_numpy()" + "ds = ekd.from_source(\"list-of-dicts\", build_lod(data)).to_fieldlist()\n", + "ds[0].shape, ds[0].geography.latlons(), ds[0].to_numpy()" ] }, { @@ -531,9 +547,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -545,7 +561,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/list_of_dicts_overview.ipynb b/docs/examples/dict/list_of_dicts_overview.ipynb similarity index 53% rename from docs/examples/list_of_dicts_overview.ipynb rename to docs/examples/dict/list_of_dicts_overview.ipynb index a9c312fbb..996f73df2 100644 --- a/docs/examples/list_of_dicts_overview.ipynb +++ b/docs/examples/dict/list_of_dicts_overview.ipynb @@ -59,20 +59,22 @@ "import earthkit.data as ekd\n", "\n", "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", + " \"geography\": {\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0]\n", + " },\n", " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", + " \"time\": {\"valid_datetime\": \"2018-08-01T09:00:00Z\"},\n", " }\n", "\n", "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 600, **prototype},\n", - " ]" + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"d\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"d\"}, \"vertical\": {\"level\": 600}, **prototype},\n", + "]" ] }, { @@ -103,7 +105,7 @@ }, "outputs": [], "source": [ - "ds = ekd.from_source(\"list-of-dicts\", d)" + "ds = ekd.from_source(\"list-of-dicts\", d).to_fieldlist()" ] }, { @@ -140,81 +142,111 @@ " \n", " \n", " \n", - " param\n", - " level\n", - " base_datetime\n", - " valid_datetime\n", - " step\n", - " number\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 1\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 2\n", " u\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 3\n", " u\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 4\n", " d\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 5\n", " d\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 600\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", "\n", "" ], "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 t 850 None 2018-08-01T09:00:00+00:00 None None\n", - "2 u 500 None 2018-08-01T09:00:00+00:00 None None\n", - "3 u 850 None 2018-08-01T09:00:00+00:00 None None\n", - "4 d 850 None 2018-08-01T09:00:00+00:00 None None\n", - "5 d 600 None 2018-08-01T09:00:00+00:00 None None" + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "2 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "3 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "4 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "5 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 850 unknown None \n", + "2 0 days 500 unknown None \n", + "3 0 days 850 unknown None \n", + "4 0 days 850 unknown None \n", + "5 0 days 600 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll \n", + "2 _distinct_ll \n", + "3 _distinct_ll \n", + "4 _distinct_ll \n", + "5 _distinct_ll " ] }, "execution_count": 3, @@ -235,12 +267,12 @@ { "data": { "text/plain": [ - "{'lat': array([[ 10., 10.],\n", + "(array([[ 10., 10.],\n", " [ 0., 0.],\n", " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", + " array([[20., 40.],\n", " [20., 40.],\n", - " [20., 40.]])}" + " [20., 40.]]))" ] }, "execution_count": 4, @@ -249,40 +281,12 @@ } ], "source": [ - "ds[0].to_latlon()" + "ds[0].geography.latlons()" ] }, { "cell_type": "code", "execution_count": 5, - "id": "bc82d798-55a5-4fd6-b692-e59c4b5df942", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([ 10., 10., 0., 0., -10., -10.]),\n", - " array([20., 40., 20., 40., 20., 40.]))" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].grid_points()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, "id": "2c6d57a0-5a10-43d6-9b69-529ac4606d05", "metadata": {}, "outputs": [ @@ -314,7 +318,7 @@ " [5, 6]]])" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -325,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "11ce5424-0e3b-40f5-9184-73c8f058cb18", "metadata": { "editable": true, @@ -341,18 +345,18 @@ "[500, 850, 500, 850, 850, 600]" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds.metadata(\"level\")" + "ds.get(\"vertical.level\")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "bbe7dd94-c445-4e9b-aafc-06ee3ef55139", "metadata": { "editable": true, @@ -365,49 +369,21 @@ { "data": { "text/plain": [ - "{'base_time': None,\n", - " 'valid_time': datetime.datetime(2018, 8, 1, 9, 0, tzinfo=tzutc())}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].metadata().datetime()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "11f1015b-7dfd-4574-9990-61b528b5eb2b", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "datetime.datetime(2018, 8, 1, 9, 0, tzinfo=tzutc())" + "datetime.datetime(2018, 8, 1, 9, 0, tzinfo=datetime.timezone.utc)" ] }, - "execution_count": 9, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds[0].metadata().valid_datetime()" + "ds[0].time.valid_datetime()" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "id": "d8942c7c-ed21-4b75-a5bb-d084207b2692", "metadata": {}, "outputs": [ @@ -432,55 +408,69 @@ " \n", " \n", " \n", - " param\n", - " level\n", - " base_datetime\n", - " valid_datetime\n", - " step\n", - " number\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 1\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", "\n", "" ], "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 t 850 None 2018-08-01T09:00:00+00:00 None None" + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 850 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll " ] }, - "execution_count": 10, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds.sel(param=\"t\").ls()" + "ds.sel({\"parameter.variable\": \"t\"}).ls()" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "id": "0d3ab9fa-24c1-46d3-900c-4e162b65f1f8", "metadata": { "editable": true, @@ -511,98 +501,128 @@ " \n", " \n", " \n", - " param\n", - " level\n", - " base_datetime\n", - " valid_datetime\n", - " step\n", - " number\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 1\n", " u\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 500\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 2\n", " d\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 600\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 3\n", " t\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 4\n", " u\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", " 5\n", " d\n", + " 2018-08-01 09:00:00+00:00\n", + " 2018-08-01 09:00:00+00:00\n", + " 0 days\n", " 850\n", + " unknown\n", " None\n", - " 2018-08-01T09:00:00+00:00\n", - " None\n", - " None\n", + " _distinct_ll\n", " \n", " \n", "\n", "" ], "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 u 500 None 2018-08-01T09:00:00+00:00 None None\n", - "2 d 600 None 2018-08-01T09:00:00+00:00 None None\n", - "3 t 850 None 2018-08-01T09:00:00+00:00 None None\n", - "4 u 850 None 2018-08-01T09:00:00+00:00 None None\n", - "5 d 850 None 2018-08-01T09:00:00+00:00 None None" + " parameter.variable time.valid_datetime time.base_datetime \\\n", + "0 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "1 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "2 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "3 t 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "4 u 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "5 d 2018-08-01 09:00:00+00:00 2018-08-01 09:00:00+00:00 \n", + "\n", + " time.step vertical.level vertical.level_type ensemble.member \\\n", + "0 0 days 500 unknown None \n", + "1 0 days 500 unknown None \n", + "2 0 days 600 unknown None \n", + "3 0 days 850 unknown None \n", + "4 0 days 850 unknown None \n", + "5 0 days 850 unknown None \n", + "\n", + " geography.grid_type \n", + "0 _distinct_ll \n", + "1 _distinct_ll \n", + "2 _distinct_ll \n", + "3 _distinct_ll \n", + "4 _distinct_ll \n", + "5 _distinct_ll " ] }, - "execution_count": 11, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds.order_by(\"level\").ls()" + "ds.order_by(\"vertical.level\").ls()" ] } ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -614,7 +634,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/dict/list_of_dicts_to_xarray.ipynb b/docs/examples/dict/list_of_dicts_to_xarray.ipynb new file mode 100644 index 000000000..292b15b35 --- /dev/null +++ b/docs/examples/dict/list_of_dicts_to_xarray.ipynb @@ -0,0 +1,1189 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ee0f0104-8077-45f1-9746-58f29b64db92", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## list-of-dict: converting to Xarray" + ] + }, + { + "cell_type": "raw", + "id": "6cadbfbf-c7af-4927-8927-c320d9160c4f", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "This example demonstrates how :ref:`data-sources-lod` fieldlists can be converted into Xarray." + ] + }, + { + "cell_type": "markdown", + "id": "2e087423-8c96-49b4-984c-f15472fa8381", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Data containing geography" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1e5ebf7a-2fc6-453a-9e14-6b04b5135810", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 248B\n",
+       "Dimensions:    (level: 2, latitude: 3, longitude: 2)\n",
+       "Coordinates:\n",
+       "  * level      (level) int64 16B 500 850\n",
+       "  * latitude   (latitude) float64 24B 10.0 0.0 -10.0\n",
+       "  * longitude  (longitude) float64 16B 20.0 40.0\n",
+       "Data variables:\n",
+       "    t          (level, latitude, longitude) float64 96B ...\n",
+       "    u          (level, latitude, longitude) float64 96B ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.8\n",
+       "    institution:  ECMWF
" + ], + "text/plain": [ + " Size: 248B\n", + "Dimensions: (level: 2, latitude: 3, longitude: 2)\n", + "Coordinates:\n", + " * level (level) int64 16B 500 850\n", + " * latitude (latitude) float64 24B 10.0 0.0 -10.0\n", + " * longitude (longitude) float64 16B 20.0 40.0\n", + "Data variables:\n", + " t (level, latitude, longitude) float64 96B ...\n", + " u (level, latitude, longitude) float64 96B ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import earthkit.data as ekd\n", + "\n", + "prototype = {\n", + " \"geography\": {\n", + " \"latitudes\": [10.0, 0.0, -10.0],\n", + " \"longitudes\": [20, 40.0]\n", + " },\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"time\": {\"valid_datetime\": \"2018-08-01T09:00:00Z\"},\n", + " }\n", + "\n", + "\n", + "\n", + "d = [\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " ]\n", + "\n", + "ds = ekd.from_source(\"list-of-dicts\", d)\n", + "ds.to_xarray()" + ] + }, + { + "cell_type": "markdown", + "id": "94b46ec8-614b-480a-8ffe-0b1dd4e344bb", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Data without geography" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7ea3d8bf-a432-4aef-94d9-5ac0c6b19503", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 208B\n",
+       "Dimensions:  (level: 2, values: 6)\n",
+       "Coordinates:\n",
+       "  * level    (level) int64 16B 500 850\n",
+       "Dimensions without coordinates: values\n",
+       "Data variables:\n",
+       "    t        (level, values) float64 96B ...\n",
+       "    u        (level, values) float64 96B ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.8\n",
+       "    institution:  ECMWF
" + ], + "text/plain": [ + " Size: 208B\n", + "Dimensions: (level: 2, values: 6)\n", + "Coordinates:\n", + " * level (level) int64 16B 500 850\n", + "Dimensions without coordinates: values\n", + "Data variables:\n", + " t (level, values) float64 96B ...\n", + " u (level, values) float64 96B ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prototype = {\n", + " \"values\": [1, 2, 3, 4, 5, 6],\n", + " \"time\": {\"valid_datetime\": \"2018-08-01T09:00:00Z\"},\n", + " }\n", + "\n", + "\n", + "d = [\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"t\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 500}, **prototype},\n", + " {\"parameter\": {\"variable\": \"u\"}, \"vertical\": {\"level\": 850}, **prototype},\n", + " ]\n", + "\n", + "ds = ekd.from_source(\"list-of-dicts\", d)\n", + "ds.to_xarray()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6463409-7686-4d90-8cab-00a04b7119bb", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/fields_from_dict_in_loop.ipynb b/docs/examples/fields_from_dict_in_loop.ipynb deleted file mode 100644 index 9bb809686..000000000 --- a/docs/examples/fields_from_dict_in_loop.ipynb +++ /dev/null @@ -1,575 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "69237222-3be8-4f6b-985c-0a8181730dfb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## Build fieldlist from dicts in a loop" - ] - }, - { - "cell_type": "markdown", - "id": "0e02de29-ccd2-4f2c-85c5-b5218179f56f", - "metadata": {}, - "source": [ - "#### Distinct latitudes/longitudes" - ] - }, - { - "cell_type": "markdown", - "id": "dd0ba727-0fcc-4c4d-a288-73f01da2921e", - "metadata": {}, - "source": [ - "In this example each field is defined by a dict using generic metadata keys. The latitudes and longitudes specified as disctinct values, the 2D field geometry will be automatically built from them." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "a17559f9-cba1-4dd4-bc30-14a9be9891ae", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", - "\n", - "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 850, **prototype},\n", - " {\"param\": \"d\", \"level\": 600, **prototype},\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "id": "2e6a5236-09d4-4b61-be4a-e4d860c52d41", - "metadata": {}, - "source": [ - "Create a fieldlist from the data above in a loop." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "1f902ec4-1eac-4f81-8fff-926a5640363c", - "metadata": {}, - "outputs": [], - "source": [ - "from earthkit.data import ArrayField\n", - "from earthkit.data import SimpleFieldList\n", - "\n", - "ds = SimpleFieldList()\n", - "for f in d:\n", - " ds.append(ArrayField(f[\"values\"], f))" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "54099091-f8c6-49d0-997e-eb2c69137dfc", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
paramlevelbase_datetimevalid_datetimestepnumber
0t500None2018-08-01T09:00:00+00:00NoneNone
1t850None2018-08-01T09:00:00+00:00NoneNone
2u500None2018-08-01T09:00:00+00:00NoneNone
3u850None2018-08-01T09:00:00+00:00NoneNone
4d850None2018-08-01T09:00:00+00:00NoneNone
5d600None2018-08-01T09:00:00+00:00NoneNone
\n", - "
" - ], - "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 t 850 None 2018-08-01T09:00:00+00:00 None None\n", - "2 u 500 None 2018-08-01T09:00:00+00:00 None None\n", - "3 u 850 None 2018-08-01T09:00:00+00:00 None None\n", - "4 d 850 None 2018-08-01T09:00:00+00:00 None None\n", - "5 d 600 None 2018-08-01T09:00:00+00:00 None None" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1b841c36-1f50-4851-b69f-53e5e0e1f130", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'lat': array([[ 10., 10.],\n", - " [ 0., 0.],\n", - " [-10., -10.]]),\n", - " 'lon': array([[20., 40.],\n", - " [20., 40.],\n", - " [20., 40.]])}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].to_latlon()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "db1a5acf-6755-42cb-b0e9-21d7335c0c97", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([ 10., 10., 0., 0., -10., -10.]),\n", - " array([20., 40., 20., 40., 20., 40.]))" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].grid_points()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "e771527b-0b8f-4ac2-b446-ab539614b48b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[[1, 2],\n", - " [3, 4],\n", - " [5, 6]],\n", - "\n", - " [[1, 2],\n", - " [3, 4],\n", - " [5, 6]],\n", - "\n", - " [[1, 2],\n", - " [3, 4],\n", - " [5, 6]],\n", - "\n", - " [[1, 2],\n", - " [3, 4],\n", - " [5, 6]],\n", - "\n", - " [[1, 2],\n", - " [3, 4],\n", - " [5, 6]],\n", - "\n", - " [[1, 2],\n", - " [3, 4],\n", - " [5, 6]]])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.to_numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "3f11ca68-d566-44d9-9ac2-5c33a5a0abab", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[500, 850, 500, 850, 850, 600]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.metadata(\"level\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "e85b74cc-7524-49ea-9c94-9cdbec2e80dc", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "datetime.datetime(2018, 8, 1, 9, 0, tzinfo=tzutc())" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].metadata().valid_datetime()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "f59740a9-e6ce-40bd-a65b-f21a272c51c3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[20.0, 10.0]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].mars_grid" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "526a4533-a274-4549-bc08-441ae19ecb5a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
paramlevelbase_datetimevalid_datetimestepnumber
0t500None2018-08-01T09:00:00+00:00NoneNone
1t850None2018-08-01T09:00:00+00:00NoneNone
\n", - "
" - ], - "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 t 850 None 2018-08-01T09:00:00+00:00 None None" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.sel(param=\"t\").ls()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "1b69116f-7852-4ef9-85b6-51d1c4227da2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
paramlevelbase_datetimevalid_datetimestepnumber
0t500None2018-08-01T09:00:00+00:00NoneNone
1u500None2018-08-01T09:00:00+00:00NoneNone
2d600None2018-08-01T09:00:00+00:00NoneNone
3t850None2018-08-01T09:00:00+00:00NoneNone
4u850None2018-08-01T09:00:00+00:00NoneNone
5d850None2018-08-01T09:00:00+00:00NoneNone
\n", - "
" - ], - "text/plain": [ - " param level base_datetime valid_datetime step number\n", - "0 t 500 None 2018-08-01T09:00:00+00:00 None None\n", - "1 u 500 None 2018-08-01T09:00:00+00:00 None None\n", - "2 d 600 None 2018-08-01T09:00:00+00:00 None None\n", - "3 t 850 None 2018-08-01T09:00:00+00:00 None None\n", - "4 u 850 None 2018-08-01T09:00:00+00:00 None None\n", - "5 d 850 None 2018-08-01T09:00:00+00:00 None None" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.order_by(\"level\").ls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d971846b-4834-4b06-98f3-bacb3925891d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev_ecc", - "language": "python", - "name": "dev_ecc" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/geojson/NUTS_RG_20M_2021_3035.geojson b/docs/examples/geojson/NUTS_RG_20M_2021_3035.geojson new file mode 120000 index 000000000..86a48918a --- /dev/null +++ b/docs/examples/geojson/NUTS_RG_20M_2021_3035.geojson @@ -0,0 +1 @@ +../NUTS_RG_20M_2021_3035.geojson \ No newline at end of file diff --git a/docs/examples/geojson_geopandas.ipynb b/docs/examples/geojson/geojson_geopandas.ipynb similarity index 95% rename from docs/examples/geojson_geopandas.ipynb rename to docs/examples/geojson/geojson_geopandas.ipynb index 6d2c83ae2..44ffbf12c 100644 --- a/docs/examples/geojson_geopandas.ipynb +++ b/docs/examples/geojson/geojson_geopandas.ipynb @@ -51,8 +51,11 @@ "outputs": [ { "data": { + "text/html": [ + "

GeoJSON file
Path: NUTS_RG_20M_2021_3035.geojson size: 2.4 MiB
Available types: geopandas, pandas, xarray

" + ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -77,15 +80,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Converting GeoJSON to list of features \n", - "\n" + "\n" ] } ], diff --git a/docs/examples/geojson/index.rst b/docs/examples/geojson/index.rst new file mode 100644 index 000000000..f45f28146 --- /dev/null +++ b/docs/examples/geojson/index.rst @@ -0,0 +1,10 @@ +.. _geojson_examples: + +GeoJSON/GeoPandas ++++++++++++++++++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + geojson_geopandas.ipynb diff --git a/docs/examples/geotiff.ipynb b/docs/examples/geotiff/geotiff.ipynb similarity index 100% rename from docs/examples/geotiff.ipynb rename to docs/examples/geotiff/geotiff.ipynb diff --git a/docs/examples/geotiff/index.rst b/docs/examples/geotiff/index.rst new file mode 100644 index 000000000..9fb910858 --- /dev/null +++ b/docs/examples/geotiff/index.rst @@ -0,0 +1,11 @@ +.. _geotiff_examples: + + +GeoTIFF ++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + geotiff.ipynb diff --git a/docs/examples/grib_array_namespace.ipynb b/docs/examples/grib/grib_array_namespace.ipynb similarity index 64% rename from docs/examples/grib_array_namespace.ipynb rename to docs/examples/grib/grib_array_namespace.ipynb index bbf3f34b6..fe8f71be9 100644 --- a/docs/examples/grib_array_namespace.ipynb +++ b/docs/examples/grib/grib_array_namespace.ipynb @@ -43,7 +43,7 @@ "source": [ "import earthkit.data as ekd\n", "ekd.download_example_file(\"test4.grib\")\n", - "ds_in = ekd.from_source(\"file\", \"test4.grib\")" + "ds_in = ekd.from_source(\"file\", \"test4.grib\").to_fieldlist()" ] }, { @@ -138,7 +138,17 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], "source": [ "!pip install torch --quiet" ] @@ -192,68 +202,58 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -262,17 +262,17 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "1 ecmf z isobaricInhPa 500 20070101 1200 0 \n", - "2 ecmf t isobaricInhPa 850 20070101 1200 0 \n", - "3 ecmf z isobaricInhPa 850 20070101 1200 0 \n", + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "1 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "2 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "3 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll " + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 500 pressure 0 regular_ll \n", + "2 850 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll " ] }, "execution_count": 6, @@ -592,7 +592,7 @@ }, { "cell_type": "markdown", - "id": "f0efdd2b-e078-43e2-936e-35db3a645a6c", + "id": "920f6b98-f0e6-4ffc-a5e4-f8f643c23f76", "metadata": { "editable": true, "slideshow": { @@ -601,38 +601,22 @@ "tags": [] }, "source": [ - "### Building array fieldlists with from_array()" + "As expected, the values in *r1* are now differing by 2 from the ones in the original fieldlist (*r*)." ] }, { - "cell_type": "raw", - "id": "3cdcbb75-86b1-4c38-b667-73ac563c8d97", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [], - "vscode": { - "languageId": "raw" - } - }, + "cell_type": "markdown", + "id": "48f6f5a4-1304-43ef-b357-563cfa071309", + "metadata": {}, "source": [ - "We can build a new array fieldlist straight from metadata and array values using :meth:`~data.core.fieldlist.FieldList.from_array`. This can be used for computations when we want to alter the values and store the result in a new FieldList." + "### Building an array fieldlist in a loop" ] }, { "cell_type": "code", "execution_count": 15, - "id": "a92db7cb-e2e1-472e-92b6-0f42360d2105", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, + "id": "a4b086a5-406d-4d1c-bd93-1edbde84bf81", + "metadata": {}, "outputs": [ { "data": { @@ -655,68 +639,58 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -725,17 +699,17 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "1 ecmf z isobaricInhPa 500 20070101 1200 0 \n", - "2 ecmf t isobaricInhPa 850 20070101 1200 0 \n", - "3 ecmf z isobaricInhPa 850 20070101 1200 0 \n", + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "1 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "2 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "3 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll " + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 500 pressure 0 regular_ll \n", + "2 850 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll " ] }, "execution_count": 15, @@ -744,26 +718,13 @@ } ], "source": [ - "md = ds.metadata()\n", - "v = ds.to_array() + 2\n", - "r1 = ekd.FieldList.from_array(v, md)\n", + "fields = []\n", + "for f in ds:\n", + " fields.append(f.set(values=f.values + 2.))\n", + "r1 = ekd.create_fieldlist(fields)\n", "r1.ls()" ] }, - { - "cell_type": "markdown", - "id": "920f6b98-f0e6-4ffc-a5e4-f8f643c23f76", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "As expected, the values in *r1* are now differing by 2 from the ones in the original fieldlist (*r*)." - ] - }, { "cell_type": "code", "execution_count": 16, @@ -792,139 +753,6 @@ "r1[0].values[:10]" ] }, - { - "cell_type": "markdown", - "id": "48f6f5a4-1304-43ef-b357-563cfa071309", - "metadata": {}, - "source": [ - "### Building an array fieldlist in a loop" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "a4b086a5-406d-4d1c-bd93-1edbde84bf81", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa5002007010112000an0regular_ll
1ecmfzisobaricInhPa5002007010112000an0regular_ll
2ecmftisobaricInhPa8502007010112000an0regular_ll
3ecmfzisobaricInhPa8502007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "1 ecmf z isobaricInhPa 500 20070101 1200 0 \n", - "2 ecmf t isobaricInhPa 850 20070101 1200 0 \n", - "3 ecmf z isobaricInhPa 850 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md = ds.metadata()\n", - "v = ds.to_array() + 2\n", - "\n", - "r1 = ekd.SimpleFieldList()\n", - "for k in range(len(md)):\n", - " r1.append(ekd.ArrayField(v[k], md[k]))\n", - "r1.ls()" - ] - }, { "cell_type": "markdown", "id": "78b1c896-a892-4192-99a1-bc9dc1cfcd4f", @@ -957,7 +785,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "9ceea7b3-5059-4378-9a26-d282caa3b74a", "metadata": { "editable": true, @@ -988,68 +816,58 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 500\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " z\n", - " isobaricInhPa\n", + " 2007-01-01 12:00:00\n", + " 2007-01-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20070101\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -1058,20 +876,20 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "1 ecmf z isobaricInhPa 500 20070101 1200 0 \n", - "2 ecmf t isobaricInhPa 850 20070101 1200 0 \n", - "3 ecmf z isobaricInhPa 850 20070101 1200 0 \n", + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "1 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "2 t 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", + "3 z 2007-01-01 12:00:00 2007-01-01 12:00:00 0 days \n", "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll " + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 500 pressure 0 regular_ll \n", + "2 850 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll " ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -1079,9 +897,41 @@ "source": [ "path = \"_from_pytroch.grib\"\n", "r1.to_target(\"file\", path)\n", - "ds1 = ekd.from_source(\"file\", path)\n", + "ds1 = ekd.from_source(\"file\", path).to_fieldlist()\n", "ds1.ls()" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "1ed4315d-0078-4e2e-b8a1-88b132f71da0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([230.04600525, 230.04600525, 230.04600525, 230.04600525,\n", + " 230.04600525, 230.04600525, 230.04600525, 230.04600525,\n", + " 230.04600525, 230.04600525])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# the modified values were correctly written to the GRIB file\n", + "ds1[0].values[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b61f22b9-b814-4b95-bb0e-06c7d28de4ab", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/examples/grib_selection.ipynb b/docs/examples/grib/grib_contents.ipynb similarity index 57% rename from docs/examples/grib_selection.ipynb rename to docs/examples/grib/grib_contents.ipynb index d0ff3abc9..276850d13 100644 --- a/docs/examples/grib_selection.ipynb +++ b/docs/examples/grib/grib_contents.ipynb @@ -10,12 +10,11 @@ "tags": [] }, "source": [ - "## GRIB: selection using metadata" + "## GRIB: listing contents" ] }, { - "attachments": {}, - "cell_type": "markdown", + "cell_type": "raw", "metadata": { "editable": true, "raw_mimetype": "text/restructuredtext", @@ -25,7 +24,7 @@ "tags": [] }, "source": [ - "We read an example GRIB file containing 18 messages and load into a fieldlist." + "We will work with a GRIB file containing 6 messages, which we get as a sample (see :ref:`from_source() `)." ] }, { @@ -38,30 +37,55 @@ }, "tags": [] }, - "outputs": [ - { - "data": { - "text/plain": [ - "18" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import earthkit.data as ekd\n", + "ds = ekd.from_source(\"sample\", \"test6.grib\").to_fieldlist()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "### Using ls(), head() and tail()" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "All these methods (see e.g.: :py:meth:`~data.core.fieldlist.FieldList.ls`) take the same set of keyword arguments:\n", "\n", - "ds = ekd.from_source(\"sample\", \"tuv_pl.grib\").to_fieldlist()\n", - "len(ds)" + "- n: number of messages to list\n", + "- keys: the metadata keys to dump\n", + "- extra_keys: extra keys of top of the default set of keys\n", + "- component: the field component(s) to use" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ - "### Using sel" + "#### ls()" ] }, { @@ -75,7 +99,7 @@ "tags": [] }, "source": [ - "With :py:meth:`~data.core.FieldList.sel` we can select a subset of messages." + "py:meth:`~data.core.fieldlist.FieldList.ls`) lists all the fields:" ] }, { @@ -127,7 +151,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -138,7 +162,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -149,7 +173,40 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", + " pressure\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 3\n", + " t\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", + " 850\n", + " pressure\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 4\n", + " u\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", + " 850\n", + " pressure\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 5\n", + " v\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", + " 850\n", " pressure\n", " 0\n", " regular_ll\n", @@ -163,11 +220,17 @@ "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 500 pressure 0 regular_ll \n", - "1 500 pressure 0 regular_ll \n", - "2 500 pressure 0 regular_ll " + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", + "2 1000 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll \n", + "4 850 pressure 0 regular_ll \n", + "5 850 pressure 0 regular_ll " ] }, "execution_count": 2, @@ -176,21 +239,22 @@ } ], "source": [ - "ds1 = ds.sel({\"vertical.level\": 500})\n", - "ds1.ls()" + "ds.ls()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Lists are accepted:" + "The number of fields can be passed as an argument:" ] }, { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -230,7 +294,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -241,51 +305,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 2\n", - " v\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 850\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 3\n", - " t\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 4\n", - " u\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 5\n", - " v\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -298,18 +318,10 @@ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 850 pressure 0 regular_ll \n", - "1 850 pressure 0 regular_ll \n", - "2 850 pressure 0 regular_ll \n", - "3 500 pressure 0 regular_ll \n", - "4 500 pressure 0 regular_ll \n", - "5 500 pressure 0 regular_ll " + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll " ] }, "execution_count": 3, @@ -318,21 +330,15 @@ } ], "source": [ - "ds1 = ds.sel({\"vertical.level\": [500, 850]})\n", - "ds1.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Slices can define closed intervals, so they are treated as inclusive of both the start and stop values, unlike normal Python indexing:" + "ds.ls(2)" ] }, { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -372,29 +378,18 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " t\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 700\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 2\n", - " t\n", + " u\n", " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -406,13 +401,11 @@ "text/plain": [ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 850 pressure 0 regular_ll \n", - "1 700 pressure 0 regular_ll \n", - "2 500 pressure 0 regular_ll " + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll " ] }, "execution_count": 4, @@ -421,52 +414,27 @@ } ], "source": [ - "ds1 = ds.sel({\"vertical.level\": slice(500, 850), \"parameter.variable\": \"t\"})\n", - "ds1.ls()" + "ds.ls(n=2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Using order_by" + "##### Keys and extra keys" ] }, { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, + "cell_type": "markdown", + "metadata": {}, "source": [ - ":py:meth:`~data.core.FieldList.order_by` returns a sorted fieldlist. First we create a smaller subset for the :py:meth:`~data.core.FieldList.order_by`\n", - "examples. " + "We can use our own list of keys instead of the default ones." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [], - "source": [ - "ds1 = ds.sel({\"vertical.level\": slice(500, 850), \n", - " \"parameter.variable\": [\"u\", \"v\"]})" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, "outputs": [ { "data": { @@ -489,124 +457,60 @@ " \n", " \n", " \n", - " parameter.variable\n", - " time.valid_datetime\n", " time.base_datetime\n", - " time.step\n", - " vertical.level\n", - " vertical.level_type\n", - " ensemble.member\n", - " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " u\n", " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 850\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", " 1\n", - " u\n", - " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", - " 0 days\n", - " 700\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", " 2\n", - " u\n", - " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", " 3\n", - " v\n", - " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", - " 0 days\n", - " 850\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", " 4\n", - " v\n", " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 700\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", " 5\n", - " v\n", " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", - " pressure\n", - " 0\n", - " regular_ll\n", " \n", " \n", "\n", "" ], "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 850 pressure 0 regular_ll \n", - "1 700 pressure 0 regular_ll \n", - "2 500 pressure 0 regular_ll \n", - "3 850 pressure 0 regular_ll \n", - "4 700 pressure 0 regular_ll \n", - "5 500 pressure 0 regular_ll " + " time.base_datetime\n", + "0 2018-08-01 12:00:00\n", + "1 2018-08-01 12:00:00\n", + "2 2018-08-01 12:00:00\n", + "3 2018-08-01 12:00:00\n", + "4 2018-08-01 12:00:00\n", + "5 2018-08-01 12:00:00" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds1.order_by(\"parameter.variable\").ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The sorting keys can be specified as a list:" + "ds.ls(keys=\"time.base_datetime\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -630,75 +534,160 @@ " \n", " \n", " \n", - " parameter.variable\n", - " time.valid_datetime\n", " time.base_datetime\n", - " time.step\n", " vertical.level\n", - " vertical.level_type\n", - " ensemble.member\n", - " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " u\n", " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 500\n", - " pressure\n", - " 0\n", - " regular_ll\n", + " 1000\n", " \n", " \n", " 1\n", - " u\n", + " 2018-08-01 12:00:00\n", + " 1000\n", + " \n", + " \n", + " 2\n", + " 2018-08-01 12:00:00\n", + " 1000\n", + " \n", + " \n", + " 3\n", + " 2018-08-01 12:00:00\n", + " 850\n", + " \n", + " \n", + " 4\n", + " 2018-08-01 12:00:00\n", + " 850\n", + " \n", + " \n", + " 5\n", + " 2018-08-01 12:00:00\n", + " 850\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " time.base_datetime vertical.level\n", + "0 2018-08-01 12:00:00 1000\n", + "1 2018-08-01 12:00:00 1000\n", + "2 2018-08-01 12:00:00 1000\n", + "3 2018-08-01 12:00:00 850\n", + "4 2018-08-01 12:00:00 850\n", + "5 2018-08-01 12:00:00 850" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.ls(keys=[\"time.base_datetime\", \"vertical.level\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The default list of keys can be extended with ``extra_keys``:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", + " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", + " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -706,6 +695,7 @@ " \n", " \n", " \n", + " \n", " \n", " \n", "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_typeparameter.units
0t2018-08-01 12:00:002018-08-01 12:00:000 days7001000pressure0regular_llkelvin
21u2018-08-01 12:00:002018-08-01 12:00:000 days8501000pressure0regular_llmeter / second
32v2018-08-01 12:00:002018-08-01 12:00:000 days5001000pressure0regular_llmeter / second
4v3t2018-08-01 12:00:002018-08-01 12:00:000 days700850pressure0regular_llkelvin
5v4u2018-08-01 12:00:002018-08-01 12:00:000 dayspressure0regular_llmeter / second
\n", @@ -713,20 +703,25 @@ ], "text/plain": [ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 500 pressure 0 regular_ll \n", - "1 700 pressure 0 regular_ll \n", - "2 850 pressure 0 regular_ll \n", - "3 500 pressure 0 regular_ll \n", - "4 700 pressure 0 regular_ll \n", - "5 850 pressure 0 regular_ll " + " vertical.level vertical.level_type ensemble.member geography.grid_type \\\n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", + "2 1000 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll \n", + "4 850 pressure 0 regular_ll \n", + "\n", + " parameter.units \n", + "0 kelvin \n", + "1 meter / second \n", + "2 meter / second \n", + "3 kelvin \n", + "4 meter / second " ] }, "execution_count": 7, @@ -735,20 +730,40 @@ } ], "source": [ - "ds1.order_by([\"parameter.variable\", \"vertical.level\"]).ls()" + "ds.head(extra_keys=\"parameter.units\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We can prescribe the actual order within a key. It only works when all the possible values are specified:" + "#### tail()" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + ":py:meth:`~data.readers.grib.index.GribFieldList.tail` displays the last 5 messages by default:" ] }, { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -788,7 +803,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -799,14 +814,14 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " u\n", + " t\n", " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", @@ -817,7 +832,7 @@ " \n", " \n", " 3\n", - " v\n", + " u\n", " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", @@ -828,22 +843,11 @@ " \n", " \n", " 4\n", - " u\n", - " 2018-08-01 12:00:00\n", - " 2018-08-01 12:00:00\n", - " 0 days\n", - " 700\n", - " pressure\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 5\n", " v\n", " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 700\n", + " 850\n", " pressure\n", " 0\n", " regular_ll\n", @@ -856,18 +860,16 @@ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "2 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "3 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "4 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 500 pressure 0 regular_ll \n", - "1 500 pressure 0 regular_ll \n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", "2 850 pressure 0 regular_ll \n", "3 850 pressure 0 regular_ll \n", - "4 700 pressure 0 regular_ll \n", - "5 700 pressure 0 regular_ll " + "4 850 pressure 0 regular_ll " ] }, "execution_count": 8, @@ -876,20 +878,40 @@ } ], "source": [ - "ds1.order_by({\"vertical.level\": [500, 850, 700]}).ls()" + "ds.tail(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Combining sel and order_by" + "#### ls()" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "By default :py:meth:`~data.readers.grib.index.GribFieldList.ls` lists all the fields:" ] }, { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -929,7 +951,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -940,7 +962,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -951,7 +973,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 850\n", + " 1000\n", " pressure\n", " 0\n", " regular_ll\n", @@ -962,7 +984,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 850\n", " pressure\n", " 0\n", " regular_ll\n", @@ -973,7 +995,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 850\n", " pressure\n", " 0\n", " regular_ll\n", @@ -984,7 +1006,7 @@ " 2018-08-01 12:00:00\n", " 2018-08-01 12:00:00\n", " 0 days\n", - " 500\n", + " 850\n", " pressure\n", " 0\n", " regular_ll\n", @@ -1003,12 +1025,12 @@ "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 850 pressure 0 regular_ll \n", - "1 850 pressure 0 regular_ll \n", - "2 850 pressure 0 regular_ll \n", - "3 500 pressure 0 regular_ll \n", - "4 500 pressure 0 regular_ll \n", - "5 500 pressure 0 regular_ll " + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", + "2 1000 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll \n", + "4 850 pressure 0 regular_ll \n", + "5 850 pressure 0 regular_ll " ] }, "execution_count": 9, @@ -1017,15 +1039,478 @@ } ], "source": [ - "ds.sel({\"vertical.level\": [500, 850]}).order_by([\"paramater.variable\"]).ls()" + "ds.ls()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The number of fields to list can also be specified:" ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.ls(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
1v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 850 pressure 0 regular_ll \n", + "1 850 pressure 0 regular_ll " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.ls(-2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inspecting fields" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The simplest ways to see a field contents is to use the acutomatic display." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Field
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
number_of_values84
array_typendarray
array_dtypefloat64
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
variablet
unitskelvin
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
valid_datetime2018-08-01 12:00:00
base_datetime2018-08-01 12:00:00
step0:00:00
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
level1000
layerNone
level_typepressure
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
member0
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
grid_spec{'grid': [30.0, 30.0], 'area': [90.0, 0.0, -90.0, 330.0]}
grid_typeregular_ll
shape(7, 12)
area[90.0, 0.0, -90.0, 330.0]
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "dev", "language": "python", - "name": "python3" + "name": "dev" }, "language_info": { "codemirror_mode": { diff --git a/docs/examples/grib_fdb_write.ipynb b/docs/examples/grib/grib_fdb_write.ipynb similarity index 72% rename from docs/examples/grib_fdb_write.ipynb rename to docs/examples/grib/grib_fdb_write.ipynb index cf2bc7858..b050459f5 100644 --- a/docs/examples/grib_fdb_write.ipynb +++ b/docs/examples/grib/grib_fdb_write.ipynb @@ -87,7 +87,7 @@ }, "outputs": [], "source": [ - "fdb_schema = \"./default_fdb_schema\"\n", + "fdb_schema = \"../default_fdb_schema\"\n", "fdb_dir = \"./_fdb\"\n", "os.makedirs(fdb_dir, exist_ok=True)\n", "\n", @@ -153,81 +153,69 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", " 1000\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " u\n", - " isobaricInhPa\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", " 1000\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " v\n", - " isobaricInhPa\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", " 1000\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 4\n", - " ecmf\n", " u\n", - " isobaricInhPa\n", + " 2018-08-01 12:00:00\n", + " 2018-08-01 12:00:00\n", + " 0 days\n", " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -236,19 +224,19 @@ "
" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 1000 20180801 1200 0 \n", - "1 ecmf u isobaricInhPa 1000 20180801 1200 0 \n", - "2 ecmf v isobaricInhPa 1000 20180801 1200 0 \n", - "3 ecmf t isobaricInhPa 850 20180801 1200 0 \n", - "4 ecmf u isobaricInhPa 850 20180801 1200 0 \n", + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll \n", - "4 an 0 regular_ll " + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", + "2 1000 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll \n", + "4 850 pressure 0 regular_ll " ] }, "execution_count": 3, @@ -258,7 +246,7 @@ ], "source": [ "ekd.download_example_file(\"tuv_pl.grib\")\n", - "ds = ekd.from_source(\"file\", \"tuv_pl.grib\")\n", + "ds = ekd.from_source(\"file\", \"tuv_pl.grib\").to_fieldlist()\n", "ds.head()" ] }, @@ -321,8 +309,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "GribField(u,500,20180801,1200,0,0)\n", - "GribField(v,500,20180801,1200,0,0)\n" + "Field(u, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 500, pressure, 0, regular_ll)\n", + "Field(v, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 500, pressure, 0, regular_ll)\n" ] } ], @@ -342,7 +330,7 @@ "}\n", "\n", "# read fields from fdb as a stream\n", - "for f in ekd.from_source(\"fdb\", request, config=config):\n", + "for f in ekd.from_source(\"fdb\", request, config=config).to_fieldlist():\n", " print(f)" ] }, @@ -357,9 +345,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -371,7 +359,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/grib_lat_lon_value_ll.ipynb b/docs/examples/grib/grib_lat_lon_value_ll.ipynb similarity index 100% rename from docs/examples/grib_lat_lon_value_ll.ipynb rename to docs/examples/grib/grib_lat_lon_value_ll.ipynb diff --git a/docs/examples/grib_lat_lon_value_rgg.ipynb b/docs/examples/grib/grib_lat_lon_value_rgg.ipynb similarity index 100% rename from docs/examples/grib_lat_lon_value_rgg.ipynb rename to docs/examples/grib/grib_lat_lon_value_rgg.ipynb diff --git a/docs/examples/grib_missing.ipynb b/docs/examples/grib/grib_missing.ipynb similarity index 100% rename from docs/examples/grib_missing.ipynb rename to docs/examples/grib/grib_missing.ipynb diff --git a/docs/examples/grib_nearest_gridpoint.ipynb b/docs/examples/grib/grib_nearest_gridpoint.ipynb similarity index 100% rename from docs/examples/grib_nearest_gridpoint.ipynb rename to docs/examples/grib/grib_nearest_gridpoint.ipynb diff --git a/docs/examples/grib/grib_overview.ipynb b/docs/examples/grib/grib_overview.ipynb new file mode 100644 index 000000000..0bce908cc --- /dev/null +++ b/docs/examples/grib/grib_overview.ipynb @@ -0,0 +1,1964 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Using GRIB data" + ] + }, + { + "attachments": {}, + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We will work with a GRIB file containing 6 messages. First we ensure the example file is available, then read the file with :ref:`from_source() `." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import earthkit.data as ekd\n", + "ekd.download_example_file(\"test6.grib\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "

GRIB file
Path: test6.grib size: 1.4 KiB
Available types: fieldlist, pandas, xarray, numpy, array

" + ], + "text/plain": [ + "GribData(path={self._reader.path})" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_in = ekd.from_source(\"file\", \"test6.grib\")\n", + "ds_in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We load the GRIB data into a fieldlist." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "ds = ds_in.to_fieldlist()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Iteration" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "A :ref:`GRIB ` data object is represented by a :py:class:`~data.readers.grib.index.GribFieldList` consisting of :py:class:`~data.core.field.Field`\\s. When used in iteration the GRIB data is automatically loaded and released when going out of scope. As a result, during the iteration only one GRIB message at a time is kept in memory:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", + "Field(u, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", + "Field(v, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", + "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n", + "Field(u, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n", + "Field(v, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n" + ] + } + ], + "source": [ + "for f in ds:\n", + " print(f)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "### Inspecting the contents" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We can use :py:meth:`~data.readers.grib.index.GribFieldList.ls` or :py:meth:`~data.readers.grib.index.GribFieldList.describe`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
3t2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
4u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
5v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll \n", + "2 1000 pressure 0 regular_ll \n", + "3 850 pressure 0 regular_ll \n", + "4 850 pressure 0 regular_ll \n", + "5 850 pressure 0 regular_ll " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.ls()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Slicing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Standard Python slicing is available. It does not involve any loading/copying of GRIB data. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g = ds[1]\n", + "g.ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1v2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll \n", + "1 1000 pressure 0 regular_ll " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g = ds[1:3]\n", + "g.ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 850 pressure 0 regular_ll " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g = ds[-1]\n", + "g.ls()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting data values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Using values" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "The :py:attr:`~data.readers.grib.codes.GribField.values` property always returns a flat array per field:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(84,)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "v = ds[0].values\n", + "v.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([272.56417847, 272.56417847, 272.56417847, 272.56417847])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "v[0:4]" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "When called on the whole fieldlist :py:attr:`~data.readers.grib.index.GribFieldList.values` returns a 2D array:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(6, 84)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "v = ds.values\n", + "v.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Using to_numpy()" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "With :py:meth:`~data.readers.grib.codes.GribField.to_numpy` the field shape is set on the array: " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(7, 12)\n", + "(7, 12)\n" + ] + } + ], + "source": [ + "v = ds[0].to_numpy()\n", + "print(v.shape)\n", + "print(ds[0].shape)" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + ":py:meth:`~data.readers.grib.index.GribFieldList.to_numpy` behaves in the same way when called on a fieldlist:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(6, 7, 12)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "v = ds.to_numpy()\n", + "v.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "### Fields and metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A fieldlist is made up of fields. We can use the automatic display to get a quick overview about what a field contains." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Field
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
number_of_values84
array_typendarray
array_dtypefloat64
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
variablet
unitskelvin
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
valid_datetime2018-08-01 12:00:00
base_datetime2018-08-01 12:00:00
step0:00:00
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
level1000
layerNone
level_typepressure
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
member0
\n", + "
\n", + " \n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "
grid_spec{'grid': [30.0, 30.0], 'area': [90.0, 0.0, -90.0, 330.0]}
grid_typeregular_ll
shape(7, 12)
area[90.0, 0.0, -90.0, 330.0]
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0]" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We use :py:meth:`~data.readers.grib.codes.GribField.get` to access metadata. It works both on individual fields slices or whole fieldlists. The supported metadata keys are format independent." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1000" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0].get(\"vertical.level\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[1000, 't'], [1000, 'u']]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0:2].get([\"vertical.level\", \"parameter.variable\"])" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We can also call :py:meth:`~data.readers.grib.index.GribFieldList.metadata` on a fieldlist:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[1000, 1000, 1000, 850, 850, 850]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.get(\"vertical.level\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The underlying ecCodes GRIB metadata keys can also be accessed." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[1000, 't', 'ecmf'], [1000, 'u', 'ecmf']]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[0:2].metadata([\"level\", \"shortName\", \"centre\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Selection" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + ":py:meth:`~data.readers.grib.index.GribFieldList.sel` offers metadata-based selection and always creates a \"view\", so no copying of GRIB data is involved." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
1v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 850 pressure 0 regular_ll \n", + "1 850 pressure 0 regular_ll " + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g = ds.sel({\"parameter.variable\": [\"u\", \"v\"], \"vertical.level\": 850})\n", + "g.ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1t2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", + "
" + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "1 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 1000 pressure 0 regular_ll \n", + "1 850 pressure 0 regular_ll " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g = ds.sel({\"parameter.variable\": \"t\"})\n", + "g.ls()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Xarray" + ] + }, + { + "cell_type": "raw", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "Coverting GRIB to Xarray is based on the earthkit Xarray engine (part of earthkit-data) ." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 4kB\n",
+       "Dimensions:    (level: 2, latitude: 7, longitude: 12)\n",
+       "Coordinates:\n",
+       "  * level      (level) int64 16B 850 1000\n",
+       "  * latitude   (latitude) float64 56B 90.0 60.0 30.0 0.0 -30.0 -60.0 -90.0\n",
+       "  * longitude  (longitude) float64 96B 0.0 30.0 60.0 90.0 ... 270.0 300.0 330.0\n",
+       "Data variables:\n",
+       "    t          (level, latitude, longitude) float64 1kB ...\n",
+       "    u          (level, latitude, longitude) float64 1kB ...\n",
+       "    v          (level, latitude, longitude) float64 1kB ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.8\n",
+       "    institution:  ECMWF
" + ], + "text/plain": [ + " Size: 4kB\n", + "Dimensions: (level: 2, latitude: 7, longitude: 12)\n", + "Coordinates:\n", + " * level (level) int64 16B 850 1000\n", + " * latitude (latitude) float64 56B 90.0 60.0 30.0 0.0 -30.0 -60.0 -90.0\n", + " * longitude (longitude) float64 96B 0.0 30.0 60.0 90.0 ... 270.0 300.0 330.0\n", + "Data variables:\n", + " t (level, latitude, longitude) float64 1kB ...\n", + " u (level, latitude, longitude) float64 1kB ...\n", + " v (level, latitude, longitude) float64 1kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.to_xarray()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/examples/numpy_fieldlist.ipynb b/docs/examples/grib/grib_selection.ipynb similarity index 54% rename from docs/examples/numpy_fieldlist.ipynb rename to docs/examples/grib/grib_selection.ipynb index 504db3d08..0d0e87618 100644 --- a/docs/examples/numpy_fieldlist.ipynb +++ b/docs/examples/grib/grib_selection.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "8f1994c5-bb2a-4c52-8405-fd240cf965ba", "metadata": { "editable": true, "slideshow": { @@ -11,12 +10,12 @@ "tags": [] }, "source": [ - "## GRIB: create fieldlist from array and metadata" + "## GRIB: selection using metadata" ] }, { - "cell_type": "raw", - "id": "a8eca176-b7fc-41d8-a1ac-e243c5f8070b", + "attachments": {}, + "cell_type": "markdown", "metadata": { "editable": true, "raw_mimetype": "text/restructuredtext", @@ -26,15 +25,12 @@ "tags": [] }, "source": [ - "In this notebook we will show how to do some computations with GRIB data and generate a array fieldlist from the results.\n", - "\n", - "First we :ref:`read ` some GRIB data containing pressure level fields." + "We read an example GRIB file containing 18 messages and load into a fieldlist." ] }, { "cell_type": "code", "execution_count": 1, - "id": "348564d3-1e60-455f-9081-499ce083d27c", "metadata": { "editable": true, "slideshow": { @@ -42,162 +38,34 @@ }, "tags": [] }, - "outputs": [], - "source": [ - "import earthkit.data as ekd\n", - "from earthkit.data import FieldList\n", - "import numpy as np\n", - "\n", - "ekd.download_example_file(\"tuv_pl.grib\")\n", - "ds = ekd.from_source(\"file\", \"tuv_pl.grib\")" - ] - }, - { - "cell_type": "markdown", - "id": "9e6a80f5-77fc-40f8-a0be-86b80275c390", - "metadata": {}, - "source": [ - "We will use the following method to compute the potential temperature:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "bce2d006-4c44-4726-82fe-2b70f7d81df2", - "metadata": {}, - "outputs": [], - "source": [ - "def potential_temperature(t, p):\n", - " # t: temperature in K\n", - " # p: pressure in Pa\n", - " return t*(100000./p)**0.285611" - ] - }, - { - "cell_type": "markdown", - "id": "03513bab-1a94-430c-b740-695f3f6c7d07", - "metadata": {}, - "source": [ - "#### Working with a single field" - ] - }, - { - "cell_type": "markdown", - "id": "ea335181-87a5-4bee-8a2e-8bd93084e75b", - "metadata": {}, - "source": [ - "In this example we compute the potential temperature for the 850 hPa level using the 4th field from the fieldlist." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "7586daef-1098-483b-9747-4aac9386e45b", - "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(GribField(t,850,20180801,1200,0,0), 'K')" + "18" ] }, - "execution_count": 3, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "f = ds[3]\n", - "f, f.metadata(\"units\")" + "import earthkit.data as ekd\n", + "\n", + "ds = ekd.from_source(\"sample\", \"tuv_pl.grib\").to_fieldlist()\n", + "len(ds)" ] }, { "cell_type": "markdown", - "id": "78801efa-c8da-4e35-abe0-33aa7d7974c0", - "metadata": {}, - "source": [ - "The computations are done with numpy arrays:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "60db0219-044d-4977-abf5-e30065400a8f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "typeOfLevel= isobaricInhPa\n" - ] - }, - { - "data": { - "text/plain": [ - "array([285.48786915, 285.48786915, 285.48786915, 285.48786915,\n", - " 285.48786915, 285.48786915, 285.48786915, 285.48786915,\n", - " 285.48786915, 285.48786915])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t = f.values\n", - "print(\"typeOfLevel=\", f.metadata(\"typeOfLevel\"))\n", - "p = f.metadata(\"level\")*100. #hPa -> Pa\n", - "t_new = potential_temperature(t, p)\n", - "t_new[:10]" - ] - }, - { - "cell_type": "raw", - "id": "07e29bb2-1410-4d4d-8feb-e6aa1af0215a", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We create a new :py:class:`~data.readers.grib.metadata.GribMetadata` object from the source field's :py:meth:`~data.readers.grib.codes.GribField.metadata` using :py:meth:`~data.readers.grib.metadata.GribMetadata.override`. The original metadata remains the same." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "f8dcc33c-6818-415d-8088-6a01666ca87f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "pt\n", - "t\n" - ] - } - ], "source": [ - "md_new = f.metadata().override(shortName=\"pt\")\n", - "print(md_new[\"shortName\"])\n", - "print(f.metadata(\"shortName\"))" + "### Using sel" ] }, { "cell_type": "raw", - "id": "058c2fd8-569b-4ddb-b8bb-217854e9ff73", "metadata": { "editable": true, "raw_mimetype": "text/restructuredtext", @@ -207,13 +75,12 @@ "tags": [] }, "source": [ - "A new FieldList (type of :py:class:`~data.indexing.fieldlist.SimpleFieldList`) can be created from the resulting ndarray and the modified metadata. It will store a list of :py:class:`~data.sources.array_list.ArrayField` fields each containing a values array and a metadata object entirely in memory. This fieldlist behaves as if it were a :py:class:`~data.readers.grib.index.GribFieldList`." + "With :py:meth:`~data.core.FieldList.sel` we can select a subset of messages." ] }, { "cell_type": "code", - "execution_count": 6, - "id": "5a2557a6-091c-4345-a792-d3bfa7876c77", + "execution_count": 2, "metadata": { "editable": true, "slideshow": { @@ -259,9 +126,35 @@ " \n", " 0\n", " ecmf\n", - " pt\n", + " t\n", " isobaricInhPa\n", - " 850\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 1\n", + " ecmf\n", + " u\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 2\n", + " ecmf\n", + " v\n", + " isobaricInhPa\n", + " 500\n", " 20180801\n", " 1200\n", " 0\n", @@ -275,69 +168,37 @@ ], "text/plain": [ " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", + "0 ecmf t isobaricInhPa 500 20180801 1200 0 \n", + "1 ecmf u isobaricInhPa 500 20180801 1200 0 \n", + "2 ecmf v isobaricInhPa 500 20180801 1200 0 \n", "\n", " dataType number gridType \n", - "0 an 0 regular_ll " + "0 an 0 regular_ll \n", + "1 an 0 regular_ll \n", + "2 an 0 regular_ll " ] }, - "execution_count": 6, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds_new = FieldList.from_array(t_new, md_new)\n", - "ds_new.ls()" + "ds1 = ds.sel({\"vertical.level\": 500})\n", + "ds1.ls()" ] }, { - "cell_type": "code", - "execution_count": 7, - "id": "2ccde1fc-f8c4-4d4d-abda-3ca3feebd02e", + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ArrayField(pt,850,20180801,1200,0,0)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_new[0]" - ] - }, - { - "cell_type": "raw", - "id": "c7733bdd-1a47-4dba-aac4-b6ed43e15337", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, "source": [ - "The new fieldlist can be saved into a GRIB file:" + "Lists are accepted:" ] }, { "cell_type": "code", - "execution_count": 8, - "id": "c22a23fc-fccd-45ef-887e-95ab61b0dee1", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, + "execution_count": 3, + "metadata": {}, "outputs": [ { "data": { @@ -376,7 +237,20 @@ " \n", " 0\n", " ecmf\n", - " pt\n", + " t\n", + " isobaricInhPa\n", + " 850\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 1\n", + " ecmf\n", + " u\n", " isobaricInhPa\n", " 850\n", " 20180801\n", @@ -386,52 +260,100 @@ " 0\n", " regular_ll\n", " \n", + " \n", + " 2\n", + " ecmf\n", + " v\n", + " isobaricInhPa\n", + " 850\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 3\n", + " ecmf\n", + " t\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 4\n", + " ecmf\n", + " u\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 5\n", + " ecmf\n", + " v\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", " \n", "\n", "" ], "text/plain": [ " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", + "0 ecmf t isobaricInhPa 850 20180801 1200 0 \n", + "1 ecmf u isobaricInhPa 850 20180801 1200 0 \n", + "2 ecmf v isobaricInhPa 850 20180801 1200 0 \n", + "3 ecmf t isobaricInhPa 500 20180801 1200 0 \n", + "4 ecmf u isobaricInhPa 500 20180801 1200 0 \n", + "5 ecmf v isobaricInhPa 500 20180801 1200 0 \n", "\n", " dataType number gridType \n", - "0 an 0 regular_ll " + "0 an 0 regular_ll \n", + "1 an 0 regular_ll \n", + "2 an 0 regular_ll \n", + "3 an 0 regular_ll \n", + "4 an 0 regular_ll \n", + "5 an 0 regular_ll " ] }, - "execution_count": 8, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "path = \"_pt_single.grib\"\n", - "ds_new.to_target(\"file\", path)\n", - "\n", - "# read file back and check content\n", - "ds1 = ekd.from_source(\"file\",path)\n", + "ds1 = ds.sel({\"vertical.level\": [500, 850]})\n", "ds1.ls()" ] }, { "cell_type": "markdown", - "id": "b717b7f9-dd46-43cc-81d7-570164954a0d", - "metadata": {}, - "source": [ - "#### Working with multiple fields" - ] - }, - { - "cell_type": "markdown", - "id": "671716f8-19cf-47e9-bb8f-45c9651bf76e", "metadata": {}, "source": [ - "In this example we compute the potential temperature for 3 pressure levels." + "Slices can define closed intervals, so they are treated as inclusive of both the start and stop values, unlike normal Python indexing:" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "bedb28e0-dc98-4145-8559-da96cadca828", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -455,114 +377,38 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", - " units\n", " \n", " \n", " \n", - " \n", - " 0\n", - " ecmf\n", - " t\n", - " isobaricInhPa\n", - " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " K\n", - " \n", - " \n", - " 1\n", - " ecmf\n", - " t\n", - " isobaricInhPa\n", - " 700\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " K\n", - " \n", - " \n", - " 2\n", - " ecmf\n", - " t\n", - " isobaricInhPa\n", - " 500\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " K\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 850 20180801 1200 0 \n", - "1 ecmf t isobaricInhPa 700 20180801 1200 0 \n", - "2 ecmf t isobaricInhPa 500 20180801 1200 0 \n", - "\n", - " dataType number gridType units \n", - "0 an 0 regular_ll K \n", - "1 an 0 regular_ll K \n", - "2 an 0 regular_ll K " + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" ] }, - "execution_count": 9, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "fs = ds.sel(shortName=\"t\", level=[850, 700, 500])\n", - "fs.ls(extra_keys=[\"units\"])" + "ds1 = ds.sel({\"vertical.level\": slice(500, 850), \"parameter.variable\": \"t\"})\n", + "ds1.ls()" ] }, { - "cell_type": "code", - "execution_count": 10, - "id": "74881517-ce58-4692-b1f1-cf281c5c1df2", + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(3, 84)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "p = np.asarray(fs.metadata(\"level\")).reshape(-1, 1)*100. # hPa -> Pa\n", - "t_new = potential_temperature(fs.values, p)\n", - "t_new.shape" + "### Using order_by" ] }, { "cell_type": "raw", - "id": "8f2f7612-3ff9-4982-9cb0-a91a426f1b0f", "metadata": { "editable": true, "raw_mimetype": "text/restructuredtext", @@ -572,13 +418,23 @@ "tags": [] }, "source": [ - "We create an array fieldlist from the resulting ndarray and the modified metadata." + ":py:meth:`~data.core.FieldList.order_by` returns a sorted fieldlist. First we create a smaller subset for the :py:meth:`~data.core.FieldList.order_by`\n", + "examples. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "ds1 = ds.sel({\"vertical.level\": slice(500, 850), \n", + " \"parameter.variable\": [\"u\", \"v\"]})" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "ab0c805e-9e33-40f1-a816-501c2cff64a1", + "execution_count": 6, "metadata": { "editable": true, "slideshow": { @@ -608,111 +464,39 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", " \n", " \n", " \n", - " \n", - " 0\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 1\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 700\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 2\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 500\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", - "1 ecmf pt isobaricInhPa 700 20180801 1200 0 \n", - "2 ecmf pt isobaricInhPa 500 20180801 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll " + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" ] }, - "execution_count": 11, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "md_new = [f.metadata().override(shortName=\"pt\") for f in fs]\n", - "ds_new = FieldList.from_array(t_new, md_new)\n", - "ds_new.ls()" + "ds1.order_by(\"parameter.variable\").ls()" ] }, { - "cell_type": "raw", - "id": "ca785b7b-d7a8-424b-99c5-19056a0b23b0", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, + "cell_type": "markdown", + "metadata": {}, "source": [ - "The new fieldlist can be saved into a GRIB file:" + "The sorting keys can be specified as a list:" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "e8e1c8a2-87aa-4ec2-895e-8ea7bd83e101", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, + "execution_count": 7, + "metadata": {}, "outputs": [ { "data": { @@ -735,145 +519,38 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", " \n", " \n", " \n", - " \n", - " 0\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 1\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 700\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 2\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 500\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", - "1 ecmf pt isobaricInhPa 700 20180801 1200 0 \n", - "2 ecmf pt isobaricInhPa 500 20180801 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll " + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" ] }, - "execution_count": 12, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "path = \"_pt_multi.grib\"\n", - "ds_new.to_target(\"file\", path)\n", - "\n", - "# read file back and check content\n", - "ds1 = ekd.from_source(\"file\", path)\n", - "ds1.ls()" + "ds1.order_by([\"parameter.variable\", \"vertical.level\"]).ls()" ] }, { "cell_type": "markdown", - "id": "3f4d3b46-e905-4ac6-88c2-bd514101d61b", "metadata": {}, "source": [ - "#### Performing the computations in a loop" - ] - }, - { - "cell_type": "raw", - "id": "4fc8aa2f-c5ea-4310-841a-98c863a4acd7", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "In this example we create an **empty** :py:class:`~data.core.fieldlist.FieldList` and add the results of the computations to it in a loop." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "6c6e8651-f5a3-4534-9227-812e8e69407d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "fs = ds.sel(shortName=\"t\", level=[850, 700, 500])\n", - "\n", - "# create an empty fieldlist\n", - "ds_r = FieldList()\n", - "\n", - "for f in fs:\n", - " p = f.metadata(\"level\")*100. # hPa -> Pa\n", - " t_new = potential_temperature(f.values, p)\n", - " md_new = f.metadata().override(shortName=\"pt\")\n", - " \n", - " # create new numpy fieldlist with a single field\n", - " ds_new = FieldList.from_array(t_new, md_new)\n", - "\n", - " # add it to the resulting fieldlist\n", - " ds_r += ds_new" + "We can prescribe the actual order within a key. It only works when all the possible values are specified:" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "32c53494-e8f4-4a22-aa93-e16b5188483a", + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -897,105 +574,38 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", " \n", " \n", " \n", - " \n", - " 0\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 850\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 1\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 700\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", - " \n", - " 2\n", - " ecmf\n", - " pt\n", - " isobaricInhPa\n", - " 500\n", - " 20180801\n", - " 1200\n", - " 0\n", - " an\n", - " 0\n", - " regular_ll\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", - "1 ecmf pt isobaricInhPa 700 20180801 1200 0 \n", - "2 ecmf pt isobaricInhPa 500 20180801 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll " + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" ] }, - "execution_count": 14, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ds_r.ls()" + "ds1.order_by({\"vertical.level\": [500, 850, 700]}).ls()" ] }, { - "cell_type": "raw", - "id": "b7635ff3-9610-415b-861d-de934591c960", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [], - "vscode": { - "languageId": "raw" - } - }, + "cell_type": "markdown", + "metadata": {}, "source": [ - "The new fieldlist can be saved into a GRIB file:" + "### Combining sel and order_by" ] }, { "cell_type": "code", - "execution_count": 15, - "id": "a149caa0-b1ba-4b8e-b2d0-6aad7ee224a6", + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -1035,7 +645,7 @@ " \n", " 0\n", " ecmf\n", - " pt\n", + " t\n", " isobaricInhPa\n", " 850\n", " 20180801\n", @@ -1048,9 +658,9 @@ " \n", " 1\n", " ecmf\n", - " pt\n", + " u\n", " isobaricInhPa\n", - " 700\n", + " 850\n", " 20180801\n", " 1200\n", " 0\n", @@ -1061,7 +671,46 @@ " \n", " 2\n", " ecmf\n", - " pt\n", + " v\n", + " isobaricInhPa\n", + " 850\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 3\n", + " ecmf\n", + " t\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 4\n", + " ecmf\n", + " u\n", + " isobaricInhPa\n", + " 500\n", + " 20180801\n", + " 1200\n", + " 0\n", + " an\n", + " 0\n", + " regular_ll\n", + " \n", + " \n", + " 5\n", + " ecmf\n", + " v\n", " isobaricInhPa\n", " 500\n", " 20180801\n", @@ -1077,44 +726,37 @@ ], "text/plain": [ " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf pt isobaricInhPa 850 20180801 1200 0 \n", - "1 ecmf pt isobaricInhPa 700 20180801 1200 0 \n", - "2 ecmf pt isobaricInhPa 500 20180801 1200 0 \n", + "0 ecmf t isobaricInhPa 850 20180801 1200 0 \n", + "1 ecmf u isobaricInhPa 850 20180801 1200 0 \n", + "2 ecmf v isobaricInhPa 850 20180801 1200 0 \n", + "3 ecmf t isobaricInhPa 500 20180801 1200 0 \n", + "4 ecmf u isobaricInhPa 500 20180801 1200 0 \n", + "5 ecmf v isobaricInhPa 500 20180801 1200 0 \n", "\n", " dataType number gridType \n", "0 an 0 regular_ll \n", "1 an 0 regular_ll \n", - "2 an 0 regular_ll " + "2 an 0 regular_ll \n", + "3 an 0 regular_ll \n", + "4 an 0 regular_ll \n", + "5 an 0 regular_ll " ] }, - "execution_count": 15, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "path = \"_pt_from_loop.grib\"\n", - "ds_r.to_target(\"file\", path)\n", - "\n", - "# read file back and check content\n", - "ds1 = ekd.from_source(\"file\", path)\n", - "ds1.ls()" + "ds.sel({\"vertical.level\": [500, 850]}).order_by([\"paramater.variable\"]).ls()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ce30259-bb20-492e-b90b-244636822dea", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "dev_ecc" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1126,9 +768,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 4 } diff --git a/docs/examples/grib_time_series.ipynb b/docs/examples/grib/grib_time_series.ipynb similarity index 100% rename from docs/examples/grib_time_series.ipynb rename to docs/examples/grib/grib_time_series.ipynb diff --git a/docs/examples/grib_to_netcdf.ipynb b/docs/examples/grib/grib_to_netcdf.ipynb similarity index 100% rename from docs/examples/grib_to_netcdf.ipynb rename to docs/examples/grib/grib_to_netcdf.ipynb diff --git a/docs/examples/grib/index.rst b/docs/examples/grib/index.rst new file mode 100644 index 000000000..a7a21e560 --- /dev/null +++ b/docs/examples/grib/index.rst @@ -0,0 +1,20 @@ +.. _grib_examples: + +GRIB +++++++ + +.. toctree:: + :maxdepth: 1 + + grib_overview.ipynb + grib_metadata.ipynb + grib_lat_lon_value_ll.ipynb + grib_lat_lon_value_rgg.ipynb + grib_selection.ipynb + grib_indexing.ipynb + grib_missing.ipynb + grib_array_namespace.ipynb + grib_nearest_gridpoint.ipynb + grib_time_series.ipynb + grib_fdb_write.ipynb + grib_to_netcdf.ipynb diff --git a/docs/examples/grib/missing.grib b/docs/examples/grib/missing.grib new file mode 120000 index 000000000..5c682bb7c --- /dev/null +++ b/docs/examples/grib/missing.grib @@ -0,0 +1 @@ +../missing.grib \ No newline at end of file diff --git a/docs/examples/grib/pl_regular_ll.grib b/docs/examples/grib/pl_regular_ll.grib new file mode 120000 index 000000000..b02cfc9a5 --- /dev/null +++ b/docs/examples/grib/pl_regular_ll.grib @@ -0,0 +1 @@ +../pl_regular_ll.grib \ No newline at end of file diff --git a/docs/examples/grib/test.grib b/docs/examples/grib/test.grib new file mode 120000 index 000000000..d7b3681fe --- /dev/null +++ b/docs/examples/grib/test.grib @@ -0,0 +1 @@ +../test.grib \ No newline at end of file diff --git a/docs/examples/grib/test4.grib b/docs/examples/grib/test4.grib new file mode 120000 index 000000000..930125f6c --- /dev/null +++ b/docs/examples/grib/test4.grib @@ -0,0 +1 @@ +../test4.grib \ No newline at end of file diff --git a/docs/examples/grib/test6.grib b/docs/examples/grib/test6.grib new file mode 120000 index 000000000..144816218 --- /dev/null +++ b/docs/examples/grib/test6.grib @@ -0,0 +1 @@ +../test6.grib \ No newline at end of file diff --git a/docs/examples/grib/time_series.grib b/docs/examples/grib/time_series.grib new file mode 120000 index 000000000..528c7422f --- /dev/null +++ b/docs/examples/grib/time_series.grib @@ -0,0 +1 @@ +../time_series.grib \ No newline at end of file diff --git a/docs/examples/grib/tuv_pl.grib b/docs/examples/grib/tuv_pl.grib new file mode 120000 index 000000000..ae0fd948a --- /dev/null +++ b/docs/examples/grib/tuv_pl.grib @@ -0,0 +1 @@ +../tuv_pl.grib \ No newline at end of file diff --git a/docs/examples/grib_contents.ipynb b/docs/examples/grib_contents.ipynb deleted file mode 100644 index 74d9788cc..000000000 --- a/docs/examples/grib_contents.ipynb +++ /dev/null @@ -1,1198 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## GRIB: listing contents" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We will work with a GRIB file containing 6 messages, which we get as a sample (see :ref:`from_source() `)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2ce22658e41540acb5e011d5ec3d53ce", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "test6.grib: 0%| | 0.00/1.41k [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
3t2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
4u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
5v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", - "" - ], - "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 1000 pressure 0 regular_ll \n", - "1 1000 pressure 0 regular_ll \n", - "2 1000 pressure 0 regular_ll \n", - "3 850 pressure 0 regular_ll \n", - "4 850 pressure 0 regular_ll \n", - "5 850 pressure 0 regular_ll " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The number of fields can be passed as an argument:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
\n", - "
" - ], - "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 1000 pressure 0 regular_ll \n", - "1 1000 pressure 0 regular_ll " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(2)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
\n", - "
" - ], - "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 1000 pressure 0 regular_ll \n", - "1 1000 pressure 0 regular_ll " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(n=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Keys and extra keys" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can use our own list of keys instead of the default ones." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
time.base_datetime
02018-08-01 12:00:00
12018-08-01 12:00:00
22018-08-01 12:00:00
32018-08-01 12:00:00
42018-08-01 12:00:00
52018-08-01 12:00:00
\n", - "
" - ], - "text/plain": [ - " time.base_datetime\n", - "0 2018-08-01 12:00:00\n", - "1 2018-08-01 12:00:00\n", - "2 2018-08-01 12:00:00\n", - "3 2018-08-01 12:00:00\n", - "4 2018-08-01 12:00:00\n", - "5 2018-08-01 12:00:00" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(keys=\"time.base_datetime\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
time.base_datetimevertical.level
02018-08-01 12:00:001000
12018-08-01 12:00:001000
22018-08-01 12:00:001000
32018-08-01 12:00:00850
42018-08-01 12:00:00850
52018-08-01 12:00:00850
\n", - "
" - ], - "text/plain": [ - " time.base_datetime vertical.level\n", - "0 2018-08-01 12:00:00 1000\n", - "1 2018-08-01 12:00:00 1000\n", - "2 2018-08-01 12:00:00 1000\n", - "3 2018-08-01 12:00:00 850\n", - "4 2018-08-01 12:00:00 850\n", - "5 2018-08-01 12:00:00 850" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(keys=[\"time.base_datetime\", \"vertical.level\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The default list of keys can be extended with ``extra_keys``:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_typeparameter.units
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_llkelvin
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_llmeter / second
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_llmeter / second
3t2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_llkelvin
4u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_llmeter / second
\n", - "
" - ], - "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \\\n", - "0 1000 pressure 0 regular_ll \n", - "1 1000 pressure 0 regular_ll \n", - "2 1000 pressure 0 regular_ll \n", - "3 850 pressure 0 regular_ll \n", - "4 850 pressure 0 regular_ll \n", - "\n", - " parameter.units \n", - "0 kelvin \n", - "1 meter / second \n", - "2 meter / second \n", - "3 kelvin \n", - "4 meter / second " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(extra_keys=\"parameter.units\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Components" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can list all the keys from one or more Field component:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "IndexFieldListBase.ls() got an unexpected keyword argument 'component'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mds\u001b[49m\u001b[43m.\u001b[49m\u001b[43mls\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcomponent\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtime\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[31mTypeError\u001b[39m: IndexFieldListBase.ls() got an unexpected keyword argument 'component'" - ] - } - ], - "source": [ - "ds.ls(component=\"time\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds.ls(component=[\"parameter\", \"time\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is possible to use any [ecCodes GRIB namespace](https://confluence.ecmwf.int/display/UDOC/What+are+namespaces+-+ecCodes+GRIB+FAQ):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "ds.ls(component=\"metadata\", filter=\"ls\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### tail()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.tail` displays the last 5 messages by default:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds.tail(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### ls()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "By default :py:meth:`~data.readers.grib.index.GribFieldList.ls` lists all the fields:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The number of fields to list can also be specified:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds.ls(2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds.ls(-2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using describe" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "With :py:meth:`~data.readers.grib.index.GribFieldList.describe` the contents of the fieldlist is presented in a concise way." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#ds.describe()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.describe` can be used with a given parameter (defined by variable or paramId):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#ds.describe(\"t\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#ds.describe(131)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting namespaces" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The field namespaces can be collected as a dict via the namespace() method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].dump(\"parameter\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "ds[0].dump([\"parameter\", \"time\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "ds[0].dump()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With this method we can also inspect any [ecCodes GRIB namespace](https://confluence.ecmwf.int/display/UDOC/What+are+namespaces+-+ecCodes+GRIB+FAQ):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].dump(part=\"metadata\", filter=\"ls\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].dump(\"metadata\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using dump" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The :py:meth:`~data.readers.grib.codes.GribField.dump` method generates a tabbed interface with the fields content in a Jupyter notebook providing some interactivity. \n", - "\n", - "By default all the field namespaces are dumped. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds[0].dump()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can dump a given namespace or namespaces." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].dump(\"parameter\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds[0].dump([\"parameter\", \"time\"])" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/x-rst", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The [ecCodes GRIB namespaces](https://confluence.ecmwf.int/display/UDOC/What+are+namespaces+-+ecCodes+GRIB+FAQ) can also be used in :py:meth:`~data.readers.grib.codes.GribField.dump`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# get all the ecCodes GRIB namespaces\n", - "ds[0].dump(\"grib\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# get a given ecCodes GRIB namespace\n", - "ds[0].dump(\"grib.ls\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/examples/grib_metadata.ipynb b/docs/examples/grib_metadata.ipynb deleted file mode 100644 index d8d47d7a5..000000000 --- a/docs/examples/grib_metadata.ipynb +++ /dev/null @@ -1,2000 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## GRIB: inspecting contents" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We will work with a GRIB file containing 6 messages. First we ensure the example file is available, then read the file with :ref:`from_source() `." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/cgr/venv/dev/lib/python3.13/site-packages/array_api_compat/torch/_info.py:361: UserWarning: 'mkldnn' is no longer used as device type. So torch.device('mkldnn') will be deprecated and removed in the future. Please use other valid device types instead. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/c10/core/Device.cpp:42.)\n", - " a = torch.empty((0,), device=torch.device(device_name, index=i))\n" - ] - } - ], - "source": [ - "import earthkit.data as ekd\n", - "ekd.download_example_file(\"test6.grib\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds = ekd.from_source(\"file\", \"test6.grib\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "### Using head(), tail() and ls()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "All these methods (:py:meth:`~data.readers.grib.index.GribFieldList.head`, :py:meth:`~data.readers.grib.index.GribFieldList.tail` and :py:meth:`~data.readers.grib.index.GribFieldList.ls`) take the same set of keyword arguments:\n", - "\n", - "- n: number of messages to list\n", - "- keys: the metadata keys to dump\n", - "- extra_keys: extra keys of top of the default set of keys\n", - "- namespace: the namespace(s) to use" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### head()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.head` displays the first 5 messages by default:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
3t2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
4u2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None \n", - "2 None None \n", - "3 None None \n", - "4 None None " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The number of fields can be passed as an argument:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(2)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(n=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Keys and extra keys" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can use our own list of keys instead of the default ones." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
base_datetime
02018-08-01 12:00:00
12018-08-01 12:00:00
22018-08-01 12:00:00
32018-08-01 12:00:00
42018-08-01 12:00:00
\n", - "
" - ], - "text/plain": [ - " base_datetime\n", - "0 2018-08-01 12:00:00\n", - "1 2018-08-01 12:00:00\n", - "2 2018-08-01 12:00:00\n", - "3 2018-08-01 12:00:00\n", - "4 2018-08-01 12:00:00" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(keys=\"base_datetime\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
base_datetimelevel
02018-08-01 12:00:001000
12018-08-01 12:00:001000
22018-08-01 12:00:001000
32018-08-01 12:00:00850
42018-08-01 12:00:00850
\n", - "
" - ], - "text/plain": [ - " base_datetime level\n", - "0 2018-08-01 12:00:00 1000\n", - "1 2018-08-01 12:00:00 1000\n", - "2 2018-08-01 12:00:00 1000\n", - "3 2018-08-01 12:00:00 850\n", - "4 2018-08-01 12:00:00 850" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(keys=[\"base_datetime\", \"level\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The default list of keys can be extended:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_typeunits
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNoneK
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNonem s**-1
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNonem s**-1
3t2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNoneK
4u2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNonem s**-1
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "\n", - " number grid_type units \n", - "0 None None K \n", - "1 None None m s**-1 \n", - "2 None None m s**-1 \n", - "3 None None K \n", - "4 None None m s**-1 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(extra_keys=\"units\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Namespaces" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The keys can be taken from an earthkit namespace:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
base_datetimevalid_datetimestep
02018-08-01 12:00:002018-08-01 12:00:000 days
12018-08-01 12:00:002018-08-01 12:00:000 days
22018-08-01 12:00:002018-08-01 12:00:000 days
32018-08-01 12:00:002018-08-01 12:00:000 days
42018-08-01 12:00:002018-08-01 12:00:000 days
\n", - "
" - ], - "text/plain": [ - " base_datetime valid_datetime step\n", - "0 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days\n", - "1 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days\n", - "2 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days\n", - "3 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days\n", - "4 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(namespace=\"time\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
base_datetimevalid_datetimestepvariableunits
02018-08-01 12:00:002018-08-01 12:00:000 daystK
12018-08-01 12:00:002018-08-01 12:00:000 daysum s**-1
22018-08-01 12:00:002018-08-01 12:00:000 daysvm s**-1
32018-08-01 12:00:002018-08-01 12:00:000 daystK
42018-08-01 12:00:002018-08-01 12:00:000 daysum s**-1
\n", - "
" - ], - "text/plain": [ - " base_datetime valid_datetime step variable units\n", - "0 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days t K\n", - "1 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days u m s**-1\n", - "2 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days v m s**-1\n", - "3 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days t K\n", - "4 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days u m s**-1" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(namespace=[\"parameter\", \"time\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is possible to use any [ecCodes GRIB namespace](https://confluence.ecmwf.int/display/UDOC/What+are+namespaces+-+ecCodes+GRIB+FAQ):" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
editioncentretypeOfLevelleveldataDatestepRangedataTypeshortNamepackingTypegridType
01ecmfisobaricInhPa1000201808010antgrid_simpleregular_ll
11ecmfisobaricInhPa1000201808010anugrid_simpleregular_ll
21ecmfisobaricInhPa1000201808010anvgrid_simpleregular_ll
31ecmfisobaricInhPa850201808010antgrid_simpleregular_ll
41ecmfisobaricInhPa850201808010anugrid_simpleregular_ll
\n", - "
" - ], - "text/plain": [ - " edition centre typeOfLevel level dataDate stepRange dataType \\\n", - "0 1 ecmf isobaricInhPa 1000 20180801 0 an \n", - "1 1 ecmf isobaricInhPa 1000 20180801 0 an \n", - "2 1 ecmf isobaricInhPa 1000 20180801 0 an \n", - "3 1 ecmf isobaricInhPa 850 20180801 0 an \n", - "4 1 ecmf isobaricInhPa 850 20180801 0 an \n", - "\n", - " shortName packingType gridType \n", - "0 t grid_simple regular_ll \n", - "1 u grid_simple regular_ll \n", - "2 v grid_simple regular_ll \n", - "3 t grid_simple regular_ll \n", - "4 u grid_simple regular_ll " - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.head(namespace=\"grib.ls\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### tail()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.tail` displays the last 5 messages by default:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1v2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
2t2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
3u2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
4v2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "2 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "3 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "4 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None \n", - "2 None None \n", - "3 None None \n", - "4 None None " - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.tail(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### ls()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "By default :py:meth:`~data.readers.grib.index.GribFieldList.ls` lists all the fields:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
3t2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
4u2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
5v2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None \n", - "2 None None \n", - "3 None None \n", - "4 None None \n", - "5 None None " - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The number of fields to list can also be specified:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 1000 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None " - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(2)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variablevalid_datetimebase_datetimesteplevellevel_typenumbergrid_type
0u2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
1v2018-08-01 12:00:002018-08-01 12:00:000 days850NoneNoneNone
\n", - "
" - ], - "text/plain": [ - " variable valid_datetime base_datetime step level level_type \\\n", - "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "1 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days 850 None \n", - "\n", - " number grid_type \n", - "0 None None \n", - "1 None None " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls(-2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using describe" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can :py:meth:`~data.readers.grib.index.GribFieldList.describe` the whole object:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
  shortNametypeOfLevelleveldatetimestepRangenumberparamIdclassstreamtypeexperimentVersionNumber
shortNametypeOfLevel            
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.describe()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can also :py:meth:`~data.readers.grib.index.GribFieldList.describe` a given parameter (defined by shortName or paramId):" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
shortName
typeOfLevel
level
date
time
stepRange
number
paramId
class
stream
type
experimentVersionNumber
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.describe(\"t\")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
shortName
typeOfLevel
level
date
time
stepRange
number
paramId
class
stream
type
experimentVersionNumber
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.describe(131)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Accessing metadata" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.metadata` with positional arguments works for both for a :py:meth:`field ` and a :py:meth:`fieldlist `. Key type qualifiers are allowed to be used." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "['pressure', 'pressure', 'pressure', 'pressure', 'pressure', 'pressure']" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.get(\"vertical_type\")" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1000, 'pressure']" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].get([\"level\", \"vertical_type\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For single fields the bracket operator can also be used:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['isobaricInhPa', 'ecmf', 98]" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].get([\"grib.typeOfLevel\", \"grib.centre\", \"grib.centre\"], astype=(None,str, int))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Namespace keys can be fetched as a dict for single fields:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].namespace(\"parameter\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Inspecting all the namespace keys for a message" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.codes.GribField.dump` gives a tabbed interface to inspect the various namespaces:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds[0].dump()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/examples/grib_metadata_object.ipynb b/docs/examples/grib_metadata_object.ipynb deleted file mode 100644 index fec2a531a..000000000 --- a/docs/examples/grib_metadata_object.ipynb +++ /dev/null @@ -1,747 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "d08d9138-0a0a-495c-b2f0-27b47e45847c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## GRIB: using the metadata object" - ] - }, - { - "cell_type": "raw", - "id": "25dd1879-9f88-4555-a847-a0ed45e3d6e4", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "In this notebook we will work with a GRIB file containing 6 messages. First we ensure the example file is available, then read the file with :ref:`from_source() `." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d1294cff-422a-4ed5-a1cf-87c6cf9195ad", - "metadata": { - "editable": true, - "raw_mimetype": "", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import earthkit.data as ekd\n", - "ekd.download_example_file(\"test6.grib\")\n", - "ds = ekd.from_source(\"file\", \"test6.grib\")" - ] - }, - { - "cell_type": "markdown", - "id": "d3392799-8f51-47f3-95b1-4a53d67a2c29", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### The metadata object" - ] - }, - { - "cell_type": "raw", - "id": "b2c90171-2ce4-445c-9993-3354286ba570", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [], - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "The recommended way to get field metadata for a given key or keys is to use :py:meth:`~data.readers.grib.codes.GribField.metadata`. See :ref:`/examples/grib_metadata.ipynb` for more examples." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "9daeae60-e906-4bd0-9055-e940218bb12c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'t'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds[0].metadata(\"shortName\")" - ] - }, - { - "cell_type": "raw", - "id": "9282df0e-2430-44fb-9025-29691bb3a24b", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Internally the field metadata is represented by a GribFieldMetadata object. Calling :py:meth:`~data.readers.grib.codes.GribField.metadata` without any arguments will return this object." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "1b795c50-f942-4417-b7e3-bd4aee4d11f5", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md = ds[0].metadata()\n", - "md" - ] - }, - { - "cell_type": "markdown", - "id": "0ad46cbe-9413-435a-bf5c-db2cb8cf8203", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "It can be used as a dict." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "5d5a9ad8-981e-440e-8327-9b21468ae42c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "('t', 1000)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md[\"shortName\"], md.get(\"level\")" - ] - }, - { - "cell_type": "markdown", - "id": "3d422ff7-c5ef-4f8d-b633-ecd93e418f3f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "
\n", - "Warning: this object does not own a GRIB handle but contains a reference to the field, which provides access to the GRIB handle. Therefore if you need to store it for later use it is recommended to create a copy with override() (see below) to decouple it from the field object.
" - ] - }, - { - "cell_type": "markdown", - "id": "e50bd099-336f-4952-941b-aa58e93f4c70", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Creating a copy of the metadata object" - ] - }, - { - "cell_type": "raw", - "id": "6cd89c6c-bcd5-4f83-b83b-03c28447ae41", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can create a copy with :py:meth:`~data.readers.grib.metadata.GribMetadata.override`, which always returns a new metadata object storing a cloned GRIB handle, thus it is decoupled from the field. " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "1369ca99-40d2-4352-8c9e-94666e5ebeec", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "md_copy = md.override()" - ] - }, - { - "cell_type": "raw", - "id": "1c511db3-f841-4936-a7cb-25dd3e3a1010", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "By default :py:meth:`~data.readers.grib.metadata.GribMetadata.override` is called with the ``headers_only_clone=True`` option to clone a new GRIB handle with all the data values (and some related information) removed. With this the resulting object can be significantly smaller, especially if the data section is large. The downside is that now the value related keys either cannot be accessed or give back wrong values. E.g when using the \"average\" key we get:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "c9ab97f0-40b7-403e-b7c9-6ce16c894d7d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(279.70703560965404, 47485.4296875)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md[\"average\"], md_copy[\"average\"]" - ] - }, - { - "cell_type": "markdown", - "id": "fe340c05-ac04-4f7d-8c43-3ed1f415a979", - "metadata": {}, - "source": [ - "To get a copy without shrinking the GRIB handle use ``headers_only_clone=False``." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "a789bb5e-b493-45cd-b7ce-e5cb8b34e125", - "metadata": {}, - "outputs": [], - "source": [ - "md_copy_full = md.override(headers_only_clone=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "3b030549-2a7e-494e-bd46-0a4cd19a09ad", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "279.70703560965404" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md_copy_full[\"average\"]" - ] - }, - { - "cell_type": "markdown", - "id": "3240cef2-baa9-4a87-a83a-dafa97b78e43", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Changing the metadata" - ] - }, - { - "cell_type": "raw", - "id": "449f7fe0-222b-456f-bb15-dbe263774f4b", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "To change metadata values pass key value pairs or a dict to :py:meth:`~data.readers.grib.metadata.GribMetadata.override`." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "40c6d232-03de-402b-82bf-8647e8a7bece", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "('z', 850)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md_mod = md.override(shortName=\"z\", level=\"850\")\n", - "md_mod[\"shortName\"], md_mod[\"level\"]" - ] - }, - { - "cell_type": "markdown", - "id": "31b017d2-843a-4dc5-8d49-b8c05f30ed37", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Since md_mod contains a clone of the GRIB handle the original metadata did not change." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "ef78a3ec-4ea2-4ff5-8c90-e60b5e07e77f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "('t', 1000)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md[\"shortName\"], md[\"level\"]" - ] - }, - { - "cell_type": "markdown", - "id": "759e4311-2419-4790-883a-84884190a8c7", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Creating array fields from metadata and values" - ] - }, - { - "cell_type": "markdown", - "id": "0316233b-8da2-49a0-b8b8-4cedd6aba6b0", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "GRIB metadata objects play a part in building new fieldlists from (altered) values and metadata." - ] - }, - { - "cell_type": "markdown", - "id": "a691f6ef-4b5b-47d0-accf-5e956702b47c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The following example computes the wind speed on 1000 hPa and creates a new fieldlist with a single field containing the new values. The metadata is taken from the u field and the shortName is modified." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "cb59ad5f-c48b-4943-984d-3abdf48fda8d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "from earthkit.data import FieldList\n", - "import numpy as np\n", - "u = ds.sel(param=\"u\", level=1000)[0]\n", - "v = ds.sel(param=\"v\", level=1000)[0]\n", - "speed = np.sqrt(u.values**2 + v.values**2)\n", - "md_speed = u.metadata().override(shortName=\"ws\")\n", - "ds_speed = FieldList.from_array(speed, md_speed) " - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "85c32bfb-c929-404f-add9-9adae40418d2", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmfwsisobaricInhPa10002018080112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf ws isobaricInhPa 1000 20180801 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_speed.ls()" - ] - }, - { - "cell_type": "raw", - "id": "a5b3c420-f71d-4a58-8cbe-761a0ce73a9b", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [], - "vscode": { - "languageId": "raw" - } - }, - "source": [ - "The resulting fieldlist contains an :py:class:`~data.sources.array_list.ArrayField`, which is composed of a numpy array storing the values and a metadata object owning its own GRIB handle with a trimmed down data section. Since the values array is decoupled from the GRIB handle stored in the metadata object, accessing metadata keys related to the data values is forbidden. Getting metadata on these keys will throw an exception. " - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "c6fe87ed-ee88-4f4d-a2b6-9401b364e2df", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_speed[0].metadata()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "27686ac4-9382-4916-ad0e-be96a649d034", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'Wind speed'" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_speed[0].metadata(\"name\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "dc28fa77-4020-431f-ad37-e480a69f9d7f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KeyError 'average'\n" - ] - } - ], - "source": [ - "try:\n", - " ds_speed[0].metadata(\"average\")\n", - "except KeyError as e:\n", - " print(f\"KeyError {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "331fa5aa-a134-423e-a4e3-bc8adf779297", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "However, strictly speaking these keys do not represent metadata and they can be easily computed from the field values when needed." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "8eab3462-3661-4fc1-9d23-8be05dc99cd8", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "7.450183054360252" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_speed[0].values.mean()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "351236ef-3b2a-4d1e-bef9-3a7154af2270", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev_ecc", - "language": "python", - "name": "dev_ecc" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/grib_modification.ipynb b/docs/examples/grib_modification.ipynb deleted file mode 100644 index 05f289d9a..000000000 --- a/docs/examples/grib_modification.ipynb +++ /dev/null @@ -1,1104 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6b0660f7-ca38-420e-b6db-64326c0b5266", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## GRIB: modifying fields" - ] - }, - { - "cell_type": "raw", - "id": "f65ece3d-feaa-4ea7-a1cb-ca0eef8056d2", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "This notebook explains how to use :py:meth:`~data.core.fieldlist.Field.clone` and :py:meth:`~data.core.fieldlist.Field.copy` to create new fields. The main difference between these methods can be summarised as follows:\n", - "\n", - "- :py:meth:`~data.core.fieldlist.Field.clone`: the new field keeps a reference to the original field and allows more flexible metadata modification than using :py:meth:`~data.core.fieldlist.Field.copy`\n", - "- :py:meth:`~data.core.fieldlist.Field.copy`: always creates a deep copy. The resulting field is always an :py:class:`~data.sources.array_list.ArrayField` storing the all the data in memory" - ] - }, - { - "cell_type": "markdown", - "id": "5b501f0b-1b9b-4c03-9cf5-7bf475d60916", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We will use the first field of the input data in the examples." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "2f3fb883-020f-4050-9026-0a3d5c553899", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa5002007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import earthkit.data as ekd\n", - "\n", - "ds = ekd.from_source(\"file\", \"test4.grib\")\n", - "f_ori = ds[0]\n", - "f_ori.ls()" - ] - }, - { - "cell_type": "markdown", - "id": "abf1ffe6-7c49-4cbc-b5c7-a267a5d6a68e", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "### Using clone()" - ] - }, - { - "cell_type": "markdown", - "id": "cfc1e1d7-9220-4e8f-ba55-ad6928432b55", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Modifying metadata" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "939aa250-f5bb-4752-82e1-2f9327c3761f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa7002007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 700 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.clone(level=700)\n", - "f_new.ls()" - ] - }, - { - "cell_type": "raw", - "id": "b49e6507-c7e5-4081-9923-990b41692fd5", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Please note that the metadata object in the new field is a wrapper around the original one and \n", - ":py:meth:`~data.readers.grib.metadata.GribMetadata.override` is not called to update it with the newly set keys. This means that there is no guarantee that the newly set keys are compatible with original metadata. So, arbitrary metadata keys can be set with no checks performed on their validity. This is generally not a problem for most of the field methods (e.g. :py:meth:`sel`) but can be an issue we we try to write the modified fields into a GRIB file (see below)." - ] - }, - { - "cell_type": "markdown", - "id": "5b47c4f6-56a3-4e30-ab33-fb00773723a3", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "To demonstrate what was said above we set \"level\" to an invalid value and add a custom (non GRIB) key:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "af960845-678a-4e7b-9eb2-3cfd1b314655", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridTypemy_key
0ecmftisobaricInhPaabc2007010112000an0regular_ll123
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa abc 20070101 1200 0 \n", - "\n", - " dataType number gridType my_key \n", - "0 an 0 regular_ll 123 " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.clone(level=\"abc\", my_key=\"123\")\n", - "f_new.ls(extra_keys=\"my_key\")" - ] - }, - { - "cell_type": "raw", - "id": "fc9105b5-4eda-4a5d-9a6b-35ce699d43e7", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Metadata values passed to :py:meth:`~data.core.fieldlist.Field.clone` can be callables." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a0b2a842-0b11-479a-8f7e-d2160e18c72e", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridTypecustom_name
0ecmftisobaricInhPa5002007010112000an0regular_llt500
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "\n", - " dataType number gridType custom_name \n", - "0 an 0 regular_ll t500 " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def _f(field, key, original_metadata):\n", - " return original_metadata[\"param\"] + str(original_metadata[\"level\"])\n", - "\n", - "f_new = f_ori.clone(custom_name=_f)\n", - "f_new.ls(extra_keys=\"custom_name\")" - ] - }, - { - "cell_type": "markdown", - "id": "5dbf44c9-8439-4da0-9caf-1c9c176c62ed", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Modifying values" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "51a4a474-afa4-48cd-b0ab-55ee295ddd92", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([229.04600525, 229.04600525, 229.04600525]),\n", - " array([228.04600525, 228.04600525, 228.04600525]))" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.clone(values=f_ori.values + 1)\n", - "f_new.values[0:3], f_ori.values[0:3]" - ] - }, - { - "cell_type": "markdown", - "id": "601f588b-1c90-431b-89de-889e55d1c521", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Modifying both metadata and values" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "0419a041-3972-415c-aab9-fbb1dbbfa642", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa7002007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 700 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.clone(values=f_ori.values + 1, level=700)\n", - "f_new.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "bf72674e-dc3f-4533-aa89-ab54e9a11ca0", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([229.04600525, 229.04600525, 229.04600525])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new.values[0:3]" - ] - }, - { - "cell_type": "markdown", - "id": "21e1dc03-4c9b-46bc-9785-1b0c6b5b2691", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Saving the modifield field into a GRIB file" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "84aaadd9-901e-4b87-a886-a197b5b504eb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/ipykernel_19855/457062347.py:1: DeprecatedWarning: save is deprecated as of 0.13.0 and will be removed in 0.14.0. Use to_target() instead\n", - " f_new.save(\"_modified_field.grib\")\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa7002007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 700 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new.save(\"_modified_field.grib\")\n", - "ds_new = ekd.from_source(\"file\", \"_modified_field.grib\")\n", - "ds_new[0].ls()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "1805823b-9904-4e17-93d3-fafbad6df6f8", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([229.04600525, 229.04600525, 229.04600525])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_new[0].values[0:3]" - ] - }, - { - "cell_type": "markdown", - "id": "01626f16-47fe-4ad7-9db4-6ce031a208e2", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Saving is not always possible" - ] - }, - { - "cell_type": "markdown", - "id": "7905e7d5-887f-4fb9-b08f-1158ae7828d3", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Saving the modified field into a GRIB is only possible if the modified metadata is GRIB compatible. To demonstrate this we add a custom metadata key called \"_level\"." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "ee9caaf2-4626-4acb-a849-0d8460af5237", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(700, 500)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.clone(_level=700)\n", - "f_new.metadata(\"_level\", \"level\")" - ] - }, - { - "cell_type": "markdown", - "id": "f280dbf1-d6a7-4d7b-83b1-bdee92aaf249", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Writing to GRIB is not possible because \"_level\" is not a valid GRIB key and ecCodes raises an exception." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5baa362d-ea7d-4b41-ace9-0631dba2f756", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Error setting _level=700\n", - "Key/value not found\n", - "Traceback (most recent call last):\n", - " File \"/Users/cgr/git/earthkit-data/src/earthkit/data/utils/message.py\", line 274, in set\n", - " return eccodes.codes_set(self._handle, name, value)\n", - " File \"/Users/cgr/git/eccodes-python/gribapi/gribapi.py\", line 2140, in grib_set\n", - " grib_set_long(msgid, key, value)\n", - " File \"/Users/cgr/git/eccodes-python/gribapi/gribapi.py\", line 1006, in grib_set_long\n", - " GRIB_CHECK(lib.grib_set_long(h, key.encode(ENC), value))\n", - " File \"/Users/cgr/git/eccodes-python/gribapi/gribapi.py\", line 226, in GRIB_CHECK\n", - " errors.raise_grib_error(errid)\n", - " File \"/Users/cgr/git/eccodes-python/gribapi/errors.py\", line 381, in raise_grib_error\n", - " raise ERROR_MAP[errid](errid)\n", - "gribapi.errors.KeyValueNotFoundError: Key/value not found\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Key/value not found\n" - ] - } - ], - "source": [ - "try:\n", - " f_new.to_target(\"file\", \"_modified_field1.grib\")\n", - "except Exception as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "id": "bce26496-7c9b-4ea4-8660-bacd5ef06ef6", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "### Using copy()" - ] - }, - { - "cell_type": "raw", - "id": "ff2c9b5a-f9ba-4c47-b3b2-db2c6790c9c0", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.core.fieldlist.Field.copy` performs a deep copy to generate an :py:class:`~data.sources.array_list.ArrayField` storing all the all the data in memory. \n", - "\n", - "When :py:meth:`~data.core.fieldlist.Field.copy` is called without arguments both the values and the Metadata object of the original field are copied into the new field. The latter copy is created by calling :py:meth:`~data.readers.grib.metadata.GribMetadata.override` on the original metadata object." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "1265f65d-c39b-4b22-ab41-3d03d532e08d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([228.04600525, 228.04600525, 228.04600525]),\n", - " array([228.04600525, 228.04600525, 228.04600525]))" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.copy()\n", - "f_new.values[:3], f_ori.values[:3]" - ] - }, - { - "cell_type": "raw", - "id": "de3c15ba-770b-4f79-a134-5086f5aad9a3", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can pass new values to :py:meth:`~data.core.fieldlist.Field.copy`." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "43e186a1-195e-424c-b778-356da142e163", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([229.04600525, 229.04600525, 229.04600525]),\n", - " array([228.04600525, 228.04600525, 228.04600525]))" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.copy(values=f_ori.values + 1)\n", - "f_new.values[:3], f_ori.values[:3]" - ] - }, - { - "cell_type": "markdown", - "id": "51aae760-50e3-44be-aff6-06cc3b5ec641", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can also pass a new metadata object." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "20fb42a2-8d98-43dd-a270-42710922b19f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa3002007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 300 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll " - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f_new = f_ori.copy(metadata=f_ori.metadata().override(level=300))\n", - "f_new.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c4d862b1-21c7-4c8c-9d80-c2510080ca01", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev_ecc", - "language": "python", - "name": "dev_ecc" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/grib_overview.ipynb b/docs/examples/grib_overview.ipynb deleted file mode 100644 index 80e478e9c..000000000 --- a/docs/examples/grib_overview.ipynb +++ /dev/null @@ -1,776 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## Using GRIB data" - ] - }, - { - "attachments": {}, - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We will work with a GRIB file containing 6 messages. First we ensure the example file is available, then read the file with :ref:`from_source() `." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import earthkit.data as ekd\n", - "ekd.download_example_file(\"test6.grib\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds_in = ekd.from_source(\"file\", \"test6.grib\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "No GRIB data is actually loaded at this point." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "ds = ds_in.to_fieldlist()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Iteration" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "A :ref:`GRIB ` data object is represented by a :py:class:`~data.readers.grib.index.GribFieldList` consisting of :py:class:`~data.core.field.Field`\\s. When used in iteration the GRIB data is automatically loaded and released when going out of scope. As a result, during the iteration only one GRIB message at a time is kept in memory:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", - "Field(u, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", - "Field(v, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 1000, pressure, 0, regular_ll)\n", - "Field(t, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n", - "Field(u, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n", - "Field(v, 2018-08-01 12:00:00, 2018-08-01 12:00:00, 0:00:00, 850, pressure, 0, regular_ll)\n" - ] - } - ], - "source": [ - "for f in ds:\n", - " print(f)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "### Inspecting the contents" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can use :py:meth:`~data.readers.grib.index.GribFieldList.ls` or :py:meth:`~data.readers.grib.index.GribFieldList.describe`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "6" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(ds)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
0t2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
1u2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
2v2018-08-01 12:00:002018-08-01 12:00:000 days1000pressure0regular_ll
3t2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
4u2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
5v2018-08-01 12:00:002018-08-01 12:00:000 days850pressure0regular_ll
\n", - "
" - ], - "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "1 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "2 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "3 t 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "4 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "5 v 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 1000 pressure 0 regular_ll \n", - "1 1000 pressure 0 regular_ll \n", - "2 1000 pressure 0 regular_ll \n", - "3 850 pressure 0 regular_ll \n", - "4 850 pressure 0 regular_ll \n", - "5 850 pressure 0 regular_ll " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "'NoneType' object is not iterable", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mds\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdescribe\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/earthkit-data/src/earthkit/data/indexing/indexed.py:270\u001b[39m, in \u001b[36mIndexFieldListBase.describe\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 266\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[32m 267\u001b[39m \u001b[38;5;66;03m# PW: TODO: `default` and `astype` kwargs of `f._get_fast()` might need to be aligned with `keys`\u001b[39;00m\n\u001b[32m 268\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m (f._get_fast(\u001b[38;5;28mself\u001b[39m._describe_keys, output=\u001b[38;5;28mdict\u001b[39m))\n\u001b[32m--> \u001b[39m\u001b[32m270\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mformat_describe\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_proc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/earthkit-data/src/earthkit/data/utils/summary.py:162\u001b[39m, in \u001b[36mformat_describe\u001b[39m\u001b[34m(attributes, *args, **kwargs)\u001b[39m\n\u001b[32m 158\u001b[39m param = kwargs.pop(\u001b[33m\"\u001b[39m\u001b[33mparam\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m 160\u001b[39m \u001b[38;5;66;03m# do_print = kwargs.pop(\"print\", True)\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m162\u001b[39m df = \u001b[43mpd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mDataFrame\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_records\u001b[49m\u001b[43m(\u001b[49m\u001b[43mattributes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 164\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m df \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m df.empty:\n\u001b[32m 165\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/venv/dev/lib/python3.13/site-packages/pandas/core/frame.py:2459\u001b[39m, in \u001b[36mDataFrame.from_records\u001b[39m\u001b[34m(cls, data, index, exclude, columns, coerce_float, nrows)\u001b[39m\n\u001b[32m 2456\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m()\n\u001b[32m 2458\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2459\u001b[39m first_row = \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2460\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[32m 2461\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m(index=index, columns=columns)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/earthkit-data/src/earthkit/data/indexing/indexed.py:268\u001b[39m, in \u001b[36mIndexFieldListBase.describe.._proc\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 265\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_proc\u001b[39m():\n\u001b[32m 266\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[32m 267\u001b[39m \u001b[38;5;66;03m# PW: TODO: `default` and `astype` kwargs of `f._get_fast()` might need to be aligned with `keys`\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m268\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m (\u001b[43mf\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_get_fast\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_describe_keys\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mdict\u001b[39;49m\u001b[43m)\u001b[49m)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/earthkit-data/src/earthkit/data/core/field.py:809\u001b[39m, in \u001b[36mField._get_fast\u001b[39m\u001b[34m(self, keys, collections, default, astype, raise_on_missing, output, flatten_dict, remapping)\u001b[39m\n\u001b[32m 800\u001b[39m result = [\n\u001b[32m 801\u001b[39m meth(k, astype=kt, default=d, raise_on_missing=raise_on_missing)\n\u001b[32m 802\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k, kt, d \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(keys, astype, default)\n\u001b[32m 803\u001b[39m ]\n\u001b[32m 804\u001b[39m \u001b[38;5;66;03m# if output is tuple:\u001b[39;00m\n\u001b[32m 805\u001b[39m \u001b[38;5;66;03m# result = tuple(result)\u001b[39;00m\n\u001b[32m 806\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 807\u001b[39m result = {\n\u001b[32m 808\u001b[39m k: meth(k, astype=kt, default=d, raise_on_missing=raise_on_missing)\n\u001b[32m--> \u001b[39m\u001b[32m809\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k, kt, d \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mzip\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mastype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdefault\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 810\u001b[39m }\n\u001b[32m 812\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m collections:\n\u001b[32m 813\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(collections, \u001b[38;5;28mstr\u001b[39m):\n", - "\u001b[31mTypeError\u001b[39m: 'NoneType' object is not iterable" - ] - } - ], - "source": [ - "ds.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Slicing" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Standard Python slicing is available. It does not involve any loading/copying of GRIB data. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "g = ds[1]\n", - "g.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "g = ds[1:3]\n", - "g.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "g = ds[-1]\n", - "g.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting data values" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Using values" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The :py:attr:`~data.readers.grib.codes.GribField.values` property always returns a flat array per field:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "v = ds[0].values\n", - "v.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "v[0:4]" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "When called on the whole fieldlist :py:attr:`~data.readers.grib.index.GribFieldList.values` returns a 2D array:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "v = ds.values\n", - "v.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Using to_numpy()" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "With :py:meth:`~data.readers.grib.codes.GribField.to_numpy` the field shape is set on the array: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "v = ds[0].to_numpy()\n", - "print(v.shape)\n", - "print(ds[0].shape)" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.to_numpy` behaves in the same way when called on a fieldlist:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "v = ds.to_numpy()\n", - "v.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "### Metadata" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We use :py:meth:`~data.readers.grib.codes.GribField.get` to access metadata. It works both on individual fields slices or whole fieldlists. The supported metadata keys are format independent." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0].get(\"level\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0:2].get([\"level\", \"variable\"])" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can also call :py:meth:`~data.readers.grib.index.GribFieldList.metadata` on a fieldlist:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "ds.get(\"level\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The underlying ecCodes GRIB metadata keys can also be accessed but we need to use the \"grib.\" prefix." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds[0:2].get([\"grib.level\", \"grib.shortName\", \"grib.centre\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Selection" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.index.GribFieldList.sel` offers metadata-based selection and always creates a \"view\", so no copying of GRIB data is involved." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "g = ds.sel(variable=[\"u\", \"v\"], level=850)\n", - "g.ls()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "g = ds.sel(variable=\"t\")\n", - "g.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Xarray" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Xarray conversion with :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` does not involve disk writing. Under the hood it uses cfgrib." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "xds = ds.to_xarray(engine=\"cfgrib\")\n", - "xds" - ] - }, - { - "cell_type": "raw", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can pass all the kwargs arguments cfgrib accepts. On top of that earthkit-data provides the :py:meth:`ignore_keys ` option to omit keys from the default set of keys used by cfgrib:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "xds = ds.to_xarray(engine=\"cfgrib\", xarray_open_dataset_kwargs={\"backend_kwargs\": {\"ignore_keys\": [\"number\"]}})\n", - "xds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/examples/grib_to_zarr_target.ipynb b/docs/examples/grib_to_zarr_target.ipynb deleted file mode 100644 index 1df45a6fc..000000000 --- a/docs/examples/grib_to_zarr_target.ipynb +++ /dev/null @@ -1,822 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8e308cd3-7f5a-4b62-bd2d-027850282c00", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## Writing GRIB data to Zarr" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "62b00621-67cd-46b0-81ef-16278a6eee18", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8d75e667605a4e4eb22967f6a3d6e9c6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "pl.grib: 0%| | 0.00/48.8k [00:00\n", - "#T_ca541 th {\n", - " text-align: left;\n", - "}\n", - "#T_ca541_row0_col0, #T_ca541_row0_col1, #T_ca541_row0_col2, #T_ca541_row0_col3, #T_ca541_row0_col4, #T_ca541_row0_col5, #T_ca541_row0_col6, #T_ca541_row0_col7, #T_ca541_row0_col8, #T_ca541_row1_col0, #T_ca541_row1_col1, #T_ca541_row1_col2, #T_ca541_row1_col3, #T_ca541_row1_col4, #T_ca541_row1_col5, #T_ca541_row1_col6, #T_ca541_row1_col7, #T_ca541_row1_col8 {\n", - " text-align: left;\n", - "}\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
  leveldatetimestepparamIdclassstreamtypeexperimentVersionNumber
shortNametypeOfLevel         
risobaricInhPa700,50020240603,202406040,12000,6157odoperfc0001
tisobaricInhPa700,50020240603,202406040,12000,6130odoperfc0001
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.describe()" - ] - }, - { - "cell_type": "markdown", - "id": "2ab4d979-7c02-42e4-9b52-8ec12e76853b", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Using to_target() on the data object" - ] - }, - { - "cell_type": "raw", - "id": "60ee891e-f0cd-426c-8f26-8155e9c25381", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We use :func:`to_target` to write the GRIB fieldlist/field into a zarr store. First, the data is converted to Xarray then :py:func:`xarray.Dataset.to_zarr` is called to generate the zarr store. We need to set the kwargs accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "e3ff25d0-bce4-4cfc-bdd0-93ca0864d08d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/Caskroom/miniforge/base/envs/dev/lib/python3.11/site-packages/zarr/api/asynchronous.py:205: UserWarning: Consolidated metadata is currently not part in the Zarr format 3 specification. It may not be supported by other zarr implementations and may change in the future.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "# with these options each field will be a separate chunk\n", - "ds.to_target(\"zarr\", \n", - " earthkit_to_xarray_kwargs={\"chunks\": {\"forecast_reference_time\": 1, \n", - " \"step\": 1, \n", - " \"level\": 1}},\n", - " xarray_to_zarr_kwargs={\"store\": \"_pl.zarr\", \"mode\": \"w\"})" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "3ffafb60-c412-4560-9bea-089143bcf85d", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
/\n",
-       "├── forecast_reference_time (4,) int64\n",
-       "├── latitude (19,) float64\n",
-       "├── level (2,) int64\n",
-       "├── longitude (36,) float64\n",
-       "├── r (4, 2, 2, 19, 36) float64\n",
-       "├── step (2,) int64\n",
-       "└── t (4, 2, 2, 19, 36) float64\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[1m/\u001b[0m\n", - "├── \u001b[1mforecast_reference_time\u001b[0m (4,) int64\n", - "├── \u001b[1mlatitude\u001b[0m (19,) float64\n", - "├── \u001b[1mlevel\u001b[0m (2,) int64\n", - "├── \u001b[1mlongitude\u001b[0m (36,) float64\n", - "├── \u001b[1mr\u001b[0m (4, 2, 2, 19, 36) float64\n", - "├── \u001b[1mstep\u001b[0m (2,) int64\n", - "└── \u001b[1mt\u001b[0m (4, 2, 2, 19, 36) float64\n" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr \n", - "root = zarr.group(\"_pl.zarr\")\n", - "root.tree()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "515bc071-d45f-48aa-abea-0cc688f4eebc", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Type : Array\n", - "Zarr format : 3\n", - "Data type : DataType.float64\n", - "Shape : (4, 2, 2, 19, 36)\n", - "Chunk shape : (1, 1, 1, 19, 36)\n", - "Order : C\n", - "Read-only : False\n", - "Store type : LocalStore\n", - "Filters : ()\n", - "Serializer : BytesCodec(endian=)\n", - "Compressors : (ZstdCodec(level=0, checksum=False),)\n", - "No. bytes : 87552 (85.5K)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root[\"t\"].info" - ] - }, - { - "cell_type": "markdown", - "id": "ef19bc33-fcc7-4b5a-83b0-ee59da4179f0", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The zarr store can be loaded to Xarray to check its content." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "706d4467-64b8-46bf-894d-346088208fa2", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/ipykernel_45349/754541422.py:2: FutureWarning: In a future version of xarray decode_timedelta will default to False rather than None. To silence this warning, set decode_timedelta to True, False, or a 'CFTimedeltaCoder' instance.\n", - " xarray.open_dataset(\"_pl.zarr\")\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 176kB\n",
-       "Dimensions:                  (step: 2, longitude: 36,\n",
-       "                              forecast_reference_time: 4, latitude: 19, level: 2)\n",
-       "Coordinates:\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * level                    (level) int64 16B 500 700\n",
-       "Data variables:\n",
-       "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
-       "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    levtype:      pl\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 176kB\n", - "Dimensions: (step: 2, longitude: 36,\n", - " forecast_reference_time: 4, latitude: 19, level: 2)\n", - "Coordinates:\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * level (level) int64 16B 500 700\n", - "Data variables:\n", - " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import xarray\n", - "xarray.open_dataset(\"_pl.zarr\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8479a12e-e907-43de-a3a6-aefb8cbfa754", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/index.rst b/docs/examples/index.rst index 35764740f..f7d60381f 100644 --- a/docs/examples/index.rst +++ b/docs/examples/index.rst @@ -3,216 +3,22 @@ Examples ============ -Here is a list of example notebooks to illustrate how to use earthkit-data. - -Data sources -++++++++++++++ - -.. toctree:: - :maxdepth: 1 - - files.ipynb - multi_files.ipynb - file_parts.ipynb - file_stream.ipynb - tar_files.ipynb - data_from_stream.ipynb - url.ipynb - url_parts.ipynb - url_stream.ipynb - netcdf_opendap.ipynb - mars.ipynb - ads.ipynb - cds.ipynb - ecmwf_open_data.ipynb - fdb.ipynb - polytope.ipynb - polytope_feature.ipynb - s3.ipynb - wekeo.ipynb - gribjump.ipynb - -GRIB -++++++ - -.. toctree:: - :maxdepth: 1 - - grib_overview.ipynb - grib_metadata.ipynb - grib_lat_lon_value_ll.ipynb - grib_lat_lon_value_rgg.ipynb - grib_selection.ipynb - grib_indexing.ipynb - grib_missing.ipynb - grib_metadata_object.ipynb - grib_modification.ipynb - numpy_fieldlist.ipynb - grib_array_namespace.ipynb - grib_nearest_gridpoint.ipynb - grib_time_series.ipynb - grib_fdb_write.ipynb - grib_to_netcdf.ipynb - - -NetCDF -++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - netcdf.ipynb - netcdf_fieldlist.ipynb - - -BUFR -++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - bufr* - -ODB -++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - odb* - -CoverageJSON -+++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - polytope_time_series.ipynb - polytope_polygon_coverage.ipynb - polytope_vertical_profile.ipynb - -GeoTIFF -+++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - geotiff.ipynb - -GeoJSON/GeoPandas -+++++++++++++++++++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - geojson_geopandas.ipynb - -Shapefile -+++++++++++++++++++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - shapefile.ipynb - -.. _examples_lod: - -UK Met Office PP file -++++++++++++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - ukmo_pp.ipynb - - -Dictionary input -++++++++++++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - fields_from_dict_in_loop.ipynb - list_of_dicts_overview - list_of_dicts_geography - list_of_dicts_to_xarray - - -Other inputs -+++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - from_object.ipynb - pandas.ipynb - - -.. _examples_xr_engine: - -Xarray engine -+++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - xarray_engine_overview.ipynb - xarray_engine_temporal.ipynb - xarray_engine_step_ranges.ipynb - xarray_engine_seasonal.ipynb - xarray_engine_level.ipynb - xarray_engine_ensemble.ipynb - xarray_engine_variable_key.ipynb - xarray_engine_mono_variable.ipynb - xarray_engine_mono_variable_remapping.ipynb - xarray_engine_field_dims.ipynb - xarray_engine_to_grib.ipynb - xarray_engine_split.ipynb - xarray_engine_squeeze.ipynb - xarray_engine_dims_as_attrs.ipynb - xarray_engine_extra_dims.ipynb - xarray_engine_remapping.ipynb - xarray_engine_holes.ipynb - xarray_engine_chunks.ipynb - xarray_engine_chunks_on_dask_cluster.ipynb - xarray_cupy.ipynb - -Targets and encoders -+++++++++++++++++++++ - -.. toctree:: - :maxdepth: 1 - :glob: - - file_target.ipynb - grib_to_file_target.ipynb - grib_to_file_pattern_target.ipynb - grib_to_fdb_target.ipynb - grib_to_geotiff.ipynb - grib_to_zarr_target.ipynb - grib_encoder.ipynb - -Miscellaneous -+++++++++++++ .. toctree:: :maxdepth: 1 - :glob: - config.ipynb - config_env_vars.ipynb - cache.ipynb - projection.ipynb - metadata.ipynb - demo_sources_plugin.ipynb + source/index.rst + grib/index.rst + netcdf/index.rst + bufr/index.rst + odb/index.rst + covjson/index.rst + geotiff/index.rst + geojson/index.rst + shapefile/index.rst + pp/index.rst + dict/index.rst + other/index.rst + xr_engine/index.rst + target/index.rst + misc/index.rst diff --git a/docs/examples/list_of_dicts_to_xarray.ipynb b/docs/examples/list_of_dicts_to_xarray.ipynb deleted file mode 100644 index 322f3097c..000000000 --- a/docs/examples/list_of_dicts_to_xarray.ipynb +++ /dev/null @@ -1,968 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "ee0f0104-8077-45f1-9746-58f29b64db92", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## list-of-dict: converting to Xarray" - ] - }, - { - "cell_type": "raw", - "id": "6cadbfbf-c7af-4927-8927-c320d9160c4f", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "This example demonstrates how :ref:`data-sources-lod` fieldlists can be converted into Xarray." - ] - }, - { - "cell_type": "markdown", - "id": "2e087423-8c96-49b4-984c-f15472fa8381", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Data containing geography" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "1e5ebf7a-2fc6-453a-9e14-6b04b5135810", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 248B\n",
-                            "Dimensions:    (levelist: 2, latitude: 3, longitude: 2)\n",
-                            "Coordinates:\n",
-                            "  * levelist   (levelist) int64 16B 500 850\n",
-                            "  * latitude   (latitude) float64 24B 10.0 0.0 -10.0\n",
-                            "  * longitude  (longitude) float64 16B 20.0 40.0\n",
-                            "Data variables:\n",
-                            "    t          (levelist, latitude, longitude) float64 96B ...\n",
-                            "    u          (levelist, latitude, longitude) float64 96B ...\n",
-                            "Attributes:\n",
-                            "    Conventions:  CF-1.8\n",
-                            "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 248B\n", - "Dimensions: (levelist: 2, latitude: 3, longitude: 2)\n", - "Coordinates:\n", - " * levelist (levelist) int64 16B 500 850\n", - " * latitude (latitude) float64 24B 10.0 0.0 -10.0\n", - " * longitude (longitude) float64 16B 20.0 40.0\n", - "Data variables:\n", - " t (levelist, latitude, longitude) float64 96B ...\n", - " u (levelist, latitude, longitude) float64 96B ...\n", - "Attributes:\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import earthkit.data as ekd\n", - "\n", - "prototype = {\n", - " \"latitudes\": [10.0, 0.0, -10.0],\n", - " \"longitudes\": [20, 40.0],\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", - "\n", - "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " ]\n", - "\n", - "ds = ekd.from_source(\"list-of-dicts\", d)\n", - "ds.to_xarray()" - ] - }, - { - "cell_type": "markdown", - "id": "94b46ec8-614b-480a-8ffe-0b1dd4e344bb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Data without geography" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "7ea3d8bf-a432-4aef-94d9-5ac0c6b19503", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 208B\n",
-                            "Dimensions:   (levelist: 2, values: 6)\n",
-                            "Coordinates:\n",
-                            "  * levelist  (levelist) int64 16B 500 850\n",
-                            "Dimensions without coordinates: values\n",
-                            "Data variables:\n",
-                            "    t         (levelist, values) float64 96B ...\n",
-                            "    u         (levelist, values) float64 96B ...\n",
-                            "Attributes:\n",
-                            "    Conventions:  CF-1.8\n",
-                            "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 208B\n", - "Dimensions: (levelist: 2, values: 6)\n", - "Coordinates:\n", - " * levelist (levelist) int64 16B 500 850\n", - "Dimensions without coordinates: values\n", - "Data variables:\n", - " t (levelist, values) float64 96B ...\n", - " u (levelist, values) float64 96B ...\n", - "Attributes:\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "prototype = {\n", - " \"values\": [1, 2, 3, 4, 5, 6],\n", - " \"valid_datetime\": \"2018-08-01T09:00:00Z\",\n", - " }\n", - "\n", - "d = [\n", - " {\"param\": \"t\", \"level\": 500, **prototype},\n", - " {\"param\": \"t\", \"level\": 850, **prototype},\n", - " {\"param\": \"u\", \"level\": 500, **prototype},\n", - " {\"param\": \"u\", \"level\": 850, **prototype},\n", - " ]\n", - "\n", - "ds = ekd.from_source(\"list-of-dicts\", d)\n", - "ds.to_xarray()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c6463409-7686-4d90-8cab-00a04b7119bb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/metadata.ipynb b/docs/examples/metadata.ipynb deleted file mode 100644 index fe355ba77..000000000 --- a/docs/examples/metadata.ipynb +++ /dev/null @@ -1,1340 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "d1a669e9-8daf-4596-ab77-05bb0a732326", - "metadata": { - "editable": true, - "raw_mimetype": "", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "# Metadata" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "94327151-7afc-46f1-84b1-07c080d65e9f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import earthkit.data as ekd" - ] - }, - { - "cell_type": "raw", - "id": "d53140b9-a2f7-4226-a650-de61ce6762ad", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Metadata of the various data formats are represented by :py:class:`~data.core.metadata.Metadata` objects, which are:\n", - "\n", - "- immutable\n", - "- behave like a dict\n", - "- can be updated with the *override()* method creating a new object\n", - "- derived from the abstract Metadata base class" - ] - }, - { - "cell_type": "markdown", - "id": "42b48cb8-7bc5-4471-83e6-a32d924ca7ff", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## RawMetadata" - ] - }, - { - "cell_type": "raw", - "id": "0d96ce25-7b3d-4571-b91d-d63be0f62a8a", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The simplest metadata type is :py:class:`~data.core.metadata.RawMetadata`, which we can create in the same way as a dict:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c0b8a49f-e181-414b-870c-748efa81f711", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "from earthkit.data.core.metadata import RawMetadata\n", - "\n", - "# from dict\n", - "md = RawMetadata({\"shortName\": \"2t\", \"perturbationNumber\": 5})\n", - "\n", - "# from list of key/value pairs\n", - "md = RawMetadata([(\"shortName\", \"2t\"), (\"perturbationNumber\", 5)])\n", - "\n", - "# from keyword arguments \n", - "md = RawMetadata(shortName=\"2t\", perturbationNumber=5)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "440a820a-ccb4-4346-81f5-4b9a7b2d67a1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RawMetadata({'shortName': '2t', 'perturbationNumber': 5})" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md" - ] - }, - { - "cell_type": "markdown", - "id": "9ddacb9a-cb10-42d2-b67e-1e392508b73b", - "metadata": {}, - "source": [ - "#### Value access" - ] - }, - { - "cell_type": "raw", - "id": "d10d7315-5eb7-4e03-8c46-e10447389d5a", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can get the value for a key using *[]* and :py:meth:`~data.core.metadata.RawMetadata.get` just like for a dict:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "fbc16f06-af69-4c6d-91e0-a6c6f9250d09", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2t\n", - "2t\n" - ] - } - ], - "source": [ - "print(md[\"shortName\"])\n", - "print(md.get(\"shortName\"))" - ] - }, - { - "cell_type": "markdown", - "id": "485344b9-afb7-401b-abc4-81c645a687ce", - "metadata": {}, - "source": [ - "When a key is not available [] raises a KeyError." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "063c7576-781c-47bf-b998-7c3df2532c09", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KeyError: 'nonExistentKey'\n" - ] - } - ], - "source": [ - "try:\n", - " md[\"nonExistentKey\"]\n", - "except KeyError as e:\n", - " print(f\"KeyError: {e}\")" - ] - }, - { - "cell_type": "raw", - "id": "7d322f02-34ef-47ab-bf67-77503ebf3d5a", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.core.metadata.RawMetadata.get` can take a default value as a second argument. When the key is not available the default value is returned. If default is not given, it defaults to None, so that this method never raises a KeyError." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "a1d2c43f-0b90-4f01-a6c8-f81f5c9d9dea", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n", - "12\n" - ] - } - ], - "source": [ - "print(md.get(\"nonExistentKey\"))\n", - "print(md.get(\"nonExistentKey\", 12))" - ] - }, - { - "cell_type": "markdown", - "id": "9aa6981a-bbb4-4988-bab8-4cacc654c32d", - "metadata": {}, - "source": [ - "#### Iteration" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "fd9574c9-adb2-4c6d-8b10-281a403b3b42", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['shortName', 'perturbationNumber']" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(md.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "45b9c42b-18d6-4b29-8f50-1e72e07f1663", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "shortName: 2t\n", - "perturbationNumber: 5\n" - ] - } - ], - "source": [ - "for k,v in md.items():\n", - " print(f\"{k}: {v}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c0758e40-2544-4866-b10e-d8a89838c31e", - "metadata": {}, - "source": [ - "#### Override" - ] - }, - { - "cell_type": "raw", - "id": "67ef7d14-131a-4de4-ba38-b630e9e9dd82", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The metadata object is immutable but we can use :py:meth:`~data.core.metadata.RawMetadata.override` to create a new object with a modified content:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "5cd1240f-e3e0-420a-babc-b2233c7d0c55", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "md1 = md.override({\"shortName\": \"2d\", \"step\": 6})" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e566d9ae-f491-42c9-bc82-9986e82800c1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RawMetadata({'shortName': '2d', 'perturbationNumber': 5, 'step': 6})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md1" - ] - }, - { - "cell_type": "markdown", - "id": "3c6ebe6f-c598-4a6b-a755-b8fc4a9ae6a1", - "metadata": {}, - "source": [ - "The original metadata object did not change:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "fae4ae6c-7fe3-4d58-98f3-127262e2357e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RawMetadata({'shortName': '2t', 'perturbationNumber': 5})" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md" - ] - }, - { - "cell_type": "markdown", - "id": "44275c78-9a94-402f-a6a9-629f14bf325b", - "metadata": {}, - "source": [ - "## GribMetadata" - ] - }, - { - "cell_type": "markdown", - "id": "cbb338e2-e1ed-4bd3-bd36-74518ff89323", - "metadata": {}, - "source": [ - "For this exercise we read a GRIB file containing 4 messages:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "88c0ec56-9d8c-4b4d-b660-24b2495d13ec", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa5002007010112000an0regular_ll
1ecmfzisobaricInhPa5002007010112000an0regular_ll
2ecmftisobaricInhPa8502007010112000an0regular_ll
3ecmfzisobaricInhPa8502007010112000an0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20070101 1200 0 \n", - "1 ecmf z isobaricInhPa 500 20070101 1200 0 \n", - "2 ecmf t isobaricInhPa 850 20070101 1200 0 \n", - "3 ecmf z isobaricInhPa 850 20070101 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 an 0 regular_ll \n", - "1 an 0 regular_ll \n", - "2 an 0 regular_ll \n", - "3 an 0 regular_ll " - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ekd.download_example_file(\"test4.grib\")\n", - "ds = ekd.from_source(\"file\", \"test4.grib\")\n", - "ds.ls()" - ] - }, - { - "cell_type": "raw", - "id": "9c3850ea-1ac6-4d0f-8c3d-5c1b6375208b", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We cannot create GRIB metadata object from a dict or :py:class:`~data.core.metadata.RawMetadata`, but can get it from a GRIB field. The resulting :py:class:`~data.readers.grib.metadata.GribMetadata` instance will store a reference to the ecCodes handle of its parent field." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "d9e78ce1-a8af-4635-8fb3-8b8c3d486710", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "md = ds[0].metadata()" - ] - }, - { - "cell_type": "raw", - "id": "9b98cfb0-a283-4ab3-9471-d805ed01eb70", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:class:`~data.readers.grib.metadata.GribMetadata` works like a dict:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "05a8b857-e3d5-40a6-8201-adafe0239c74", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "t\n", - "t\n", - "None\n", - "12\n" - ] - } - ], - "source": [ - "print(md[\"shortName\"])\n", - "print(md.get(\"shortName\"))\n", - "print(md.get(\"nonExistentKey\"))\n", - "print(md.get(\"nonExistentKey\", 12))" - ] - }, - { - "cell_type": "raw", - "id": "8bd7cd30-9c59-44e3-a845-a6f41dff9236", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "However, it is also aware of the :xref:`eccodes_namespace`\\s, when we use :py:meth:`~data.readers.grib.metadata.GribMetadata.as_namespace` or :py:meth:`~data.readers.grib.metadata.GribMetadata.dump`. " - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "f3c0c9da-0c41-4c39-8094-b2b8a50f1680", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'typeOfLevel': 'isobaricInhPa', 'level': 500}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md.as_namespace(\"vertical\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "6fcf977a-f1e5-44ba-8d0b-549c018b3551", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
GribFieldMetadata
\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
globalDomaing
GRIBEditionNumber1
eps0
offsetSection00
section0Length8
totalLength130428
editionNumber1
WMO0
productionStatusOfProcessedData0
section1Length52
wrongPadding0
table2Version128
centreecmf
centreDescriptionEuropean Centre for Medium-Range Weather Forecasts
generatingProcessIdentifier145
gridDefinition255
indicatorOfParameter130
parameterNameTemperature
parameterUnitsK
indicatorOfTypeOfLevelpl
pressureUnitshPa
typeOfLevelECMFisobaricInhPa
typeOfLevelisobaricInhPa
level500
yearOfCentury7
month1
day1
hour12
minute0
second0
unitOfTimeRange1
P10
P20
timeRangeIndicator0
numberIncludedInAverage0
numberMissingFromAveragesOrAccumulations0
centuryOfReferenceTimeOfData21
subCentre0
paramIdECMF130
paramId130
cfNameECMFair_temperature
cfNameair_temperature
unitsECMFK
unitsK
nameECMFTemperature
nameTemperature
decimalScaleFactor0
setLocalDefinition0
optimizeScaleFactor0
dataDate20070101
year2007
dataTime1200
julianDay2454102.0
stepUnits1
stepTypeinstant
stepRange0
startStep0
endStep0
marsParam130.128
validityDate20070101
validityTime1200
validityDateTime2454101.5083333333
deleteLocalDefinition0
localUsePresent1
reservedNeedNotBePresent['']
localDefinitionNumber1
GRIBEXSection1Problem0
marsClassea
marsTypean
marsStreamoper
experimentVersionNumber0001
perturbationNumber0
numberOfForecastsInEnsemble0
padding_local1_100
grib2LocalSectionNumber1
localExtensionPadding
_xNone
section1Padding
shortNameECMFt
shortNamet
cfVarNameECMFt
cfVarNamet
ifsParam130
stepTypeForConversionunknown
md5Section1e4c6d9f0d01b01247d9c57dad662ee30
md5Producted3159ec68703fb0415e15e859b4eb02
paramIdForConversion0
gridDescriptionSectionPresent1
bitmapPresent0
angleSubdivisions1000
section2Length32
radius6367470
numberOfVerticalCoordinateValues0
neitherPresent255
pvlLocation255
dataRepresentationType0
gridDefinitionDescriptionLatitude/Longitude Grid
gridDefinitionTemplateNumber0
Ni360
Nj181
latitudeOfFirstGridPoint90000
latitudeOfFirstGridPointInDegrees90.0
longitudeOfFirstGridPoint0
longitudeOfFirstGridPointInDegrees0.0
resolutionAndComponentFlags128
ijDirectionIncrementGiven1
earthIsOblate0
resolutionAndComponentFlags30
resolutionAndComponentFlags40
uvRelativeToGrid0
resolutionAndComponentFlags60
resolutionAndComponentFlags70
resolutionAndComponentFlags80
latitudeOfLastGridPoint-90000
latitudeOfLastGridPointInDegrees-90.0
longitudeOfLastGridPoint359000
longitudeOfLastGridPointInDegrees359.0
iDirectionIncrement1000
jDirectionIncrement1000
isGridded1
scanningMode0
iScansNegatively0
jScansPositively0
jPointsAreConsecutive0
alternativeRowScanning0
iScansPositively1
jScansNegatively1
scanningMode40
scanningMode50
scanningMode60
scanningMode70
scanningMode80
swapScanningAlternativeRows0
jDirectionIncrementInDegrees1.0
iDirectionIncrementInDegrees1.0
numberOfDataPoints65160
numberOfValues65160
zeros
PVPresent0
padding_sec2_2
PLPresent0
padding_sec2_1
deletePV1
padding_sec2_3
md5Section2d0ccb07e4b36a8911817cc07539cf859
isSpectral0
lengthOfHeaders85
md5Headersb2ff552cc12a4ea08bfbd989ed20c0fb
missingValue9999
tableReference0
section4Length130332
halfByte8
dataFlag8
binaryScaleFactor-10
referenceValue224.05772399902344
referenceValueError1.52587890625e-05
sphericalHarmonics0
complexPacking0
integerPointValues0
additionalFlagPresent0
orderOfSPD2
boustrophedonic0
hideThis0
packingTypegrid_simple
bitsPerValue16
constantFieldHalfByte8
bitMapIndicator255
numberOfCodedValues65160
packingError0.00049591064453125
unpackedError1.52587890625e-05
maximum273.33799743652344
minimum224.05772399902344
average252.5098487718183
standardDeviation13.372886915179645
skewness-0.2209591997514728
kurtosis-1.2738579926507174
isConstant0.0
numberOfMissing0
dataLength16290
changeDecimalPrecision0
decimalPrecision0
bitsPerValueAndRepack16
setPackingTypegrid_simple
scaleValuesBy1.0
offsetValuesBy0.0
gridTyperegular_ll
getNumberOfValues65160
padding_sec4_1
md5Section473097e43c94a4e2947a55799222424c2
section5Length4
77777777
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
edition1
centreecmf
typeOfLevelisobaricInhPa
level500
dataDate20070101
stepRange0
dataTypean
shortNamet
packingTypegrid_simple
gridTyperegular_ll
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
bitmapPresent0
Ni360
Nj181
latitudeOfFirstGridPointInDegrees90.0
longitudeOfFirstGridPointInDegrees0.0
latitudeOfLastGridPointInDegrees-90.0
longitudeOfLastGridPointInDegrees359.0
iScansNegatively0
jScansPositively0
jPointsAreConsecutive0
jDirectionIncrementInDegrees1.0
iDirectionIncrementInDegrees1.0
gridTyperegular_ll
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
domaing
levtypepl
levelist500
date20070101
time1200
step0
paramt
classea
typean
streamoper
expver0001
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
centreecmf
paramId130
unitsK
nameTemperature
shortNamet
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
max273.33799743652344
min224.05772399902344
avg252.5098487718183
sd13.372886915179645
skew-0.2209591997514728
kurt-1.2738579926507174
const0.0
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
dataDate20070101
dataTime1200
stepUnits1
stepTypeinstant
stepRange0
startStep0
endStep0
validityDate20070101
validityTime1200
\n", - "
\n", - " \n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
typeOfLevelisobaricInhPa
level500
\n", - "
\n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " " - ], - "text/plain": [ - "{'default': {'globalDomain': 'g',\n", - " 'GRIBEditionNumber': 1,\n", - " 'eps': 0,\n", - " 'offsetSection0': 0,\n", - " 'section0Length': 8,\n", - " 'totalLength': 130428,\n", - " 'editionNumber': 1,\n", - " 'WMO': 0,\n", - " 'productionStatusOfProcessedData': 0,\n", - " 'section1Length': 52,\n", - " 'wrongPadding': 0,\n", - " 'table2Version': 128,\n", - " 'centre': 'ecmf',\n", - " 'centreDescription': 'European Centre for Medium-Range Weather Forecasts',\n", - " 'generatingProcessIdentifier': 145,\n", - " 'gridDefinition': 255,\n", - " 'indicatorOfParameter': 130,\n", - " 'parameterName': 'Temperature',\n", - " 'parameterUnits': 'K',\n", - " 'indicatorOfTypeOfLevel': 'pl',\n", - " 'pressureUnits': 'hPa',\n", - " 'typeOfLevelECMF': 'isobaricInhPa',\n", - " 'typeOfLevel': 'isobaricInhPa',\n", - " 'level': 500,\n", - " 'yearOfCentury': 7,\n", - " 'month': 1,\n", - " 'day': 1,\n", - " 'hour': 12,\n", - " 'minute': 0,\n", - " 'second': 0,\n", - " 'unitOfTimeRange': 1,\n", - " 'P1': 0,\n", - " 'P2': 0,\n", - " 'timeRangeIndicator': 0,\n", - " 'numberIncludedInAverage': 0,\n", - " 'numberMissingFromAveragesOrAccumulations': 0,\n", - " 'centuryOfReferenceTimeOfData': 21,\n", - " 'subCentre': 0,\n", - " 'paramIdECMF': '130',\n", - " 'paramId': 130,\n", - " 'cfNameECMF': 'air_temperature',\n", - " 'cfName': 'air_temperature',\n", - " 'unitsECMF': 'K',\n", - " 'units': 'K',\n", - " 'nameECMF': 'Temperature',\n", - " 'name': 'Temperature',\n", - " 'decimalScaleFactor': 0,\n", - " 'setLocalDefinition': 0,\n", - " 'optimizeScaleFactor': 0,\n", - " 'dataDate': 20070101,\n", - " 'year': 2007,\n", - " 'dataTime': 1200,\n", - " 'julianDay': 2454102.0,\n", - " 'stepUnits': 1,\n", - " 'stepType': 'instant',\n", - " 'stepRange': '0',\n", - " 'startStep': 0,\n", - " 'endStep': 0,\n", - " 'marsParam': '130.128',\n", - " 'validityDate': 20070101,\n", - " 'validityTime': 1200,\n", - " 'validityDateTime': 2454101.5083333333,\n", - " 'deleteLocalDefinition': 0,\n", - " 'localUsePresent': 1,\n", - " 'reservedNeedNotBePresent': [''],\n", - " 'localDefinitionNumber': 1,\n", - " 'GRIBEXSection1Problem': 0,\n", - " 'marsClass': 'ea',\n", - " 'marsType': 'an',\n", - " 'marsStream': 'oper',\n", - " 'experimentVersionNumber': '0001',\n", - " 'perturbationNumber': 0,\n", - " 'numberOfForecastsInEnsemble': 0,\n", - " 'padding_local1_1': '00',\n", - " 'grib2LocalSectionNumber': 1,\n", - " 'localExtensionPadding': '',\n", - " '_x': None,\n", - " 'section1Padding': '',\n", - " 'shortNameECMF': 't',\n", - " 'shortName': 't',\n", - " 'cfVarNameECMF': 't',\n", - " 'cfVarName': 't',\n", - " 'ifsParam': 130,\n", - " 'stepTypeForConversion': 'unknown',\n", - " 'md5Section1': 'e4c6d9f0d01b01247d9c57dad662ee30',\n", - " 'md5Product': 'ed3159ec68703fb0415e15e859b4eb02',\n", - " 'paramIdForConversion': 0,\n", - " 'gridDescriptionSectionPresent': 1,\n", - " 'bitmapPresent': 0,\n", - " 'angleSubdivisions': 1000,\n", - " 'section2Length': 32,\n", - " 'radius': 6367470,\n", - " 'numberOfVerticalCoordinateValues': 0,\n", - " 'neitherPresent': 255,\n", - " 'pvlLocation': 255,\n", - " 'dataRepresentationType': 0,\n", - " 'gridDefinitionDescription': 'Latitude/Longitude Grid',\n", - " 'gridDefinitionTemplateNumber': 0,\n", - " 'Ni': 360,\n", - " 'Nj': 181,\n", - " 'latitudeOfFirstGridPoint': 90000,\n", - " 'latitudeOfFirstGridPointInDegrees': 90.0,\n", - " 'longitudeOfFirstGridPoint': 0,\n", - " 'longitudeOfFirstGridPointInDegrees': 0.0,\n", - " 'resolutionAndComponentFlags': 128,\n", - " 'ijDirectionIncrementGiven': 1,\n", - " 'earthIsOblate': 0,\n", - " 'resolutionAndComponentFlags3': 0,\n", - " 'resolutionAndComponentFlags4': 0,\n", - " 'uvRelativeToGrid': 0,\n", - " 'resolutionAndComponentFlags6': 0,\n", - " 'resolutionAndComponentFlags7': 0,\n", - " 'resolutionAndComponentFlags8': 0,\n", - " 'latitudeOfLastGridPoint': -90000,\n", - " 'latitudeOfLastGridPointInDegrees': -90.0,\n", - " 'longitudeOfLastGridPoint': 359000,\n", - " 'longitudeOfLastGridPointInDegrees': 359.0,\n", - " 'iDirectionIncrement': 1000,\n", - " 'jDirectionIncrement': 1000,\n", - " 'isGridded': 1,\n", - " 'scanningMode': 0,\n", - " 'iScansNegatively': 0,\n", - " 'jScansPositively': 0,\n", - " 'jPointsAreConsecutive': 0,\n", - " 'alternativeRowScanning': 0,\n", - " 'iScansPositively': 1,\n", - " 'jScansNegatively': 1,\n", - " 'scanningMode4': 0,\n", - " 'scanningMode5': 0,\n", - " 'scanningMode6': 0,\n", - " 'scanningMode7': 0,\n", - " 'scanningMode8': 0,\n", - " 'swapScanningAlternativeRows': 0,\n", - " 'jDirectionIncrementInDegrees': 1.0,\n", - " 'iDirectionIncrementInDegrees': 1.0,\n", - " 'numberOfDataPoints': 65160,\n", - " 'numberOfValues': 65160,\n", - " 'zeros': '',\n", - " 'PVPresent': 0,\n", - " 'padding_sec2_2': '',\n", - " 'PLPresent': 0,\n", - " 'padding_sec2_1': '',\n", - " 'deletePV': '1',\n", - " 'padding_sec2_3': '',\n", - " 'md5Section2': 'd0ccb07e4b36a8911817cc07539cf859',\n", - " 'isSpectral': 0,\n", - " 'lengthOfHeaders': 85,\n", - " 'md5Headers': 'b2ff552cc12a4ea08bfbd989ed20c0fb',\n", - " 'missingValue': 9999,\n", - " 'tableReference': 0,\n", - " 'section4Length': 130332,\n", - " 'halfByte': 8,\n", - " 'dataFlag': 8,\n", - " 'binaryScaleFactor': -10,\n", - " 'referenceValue': 224.05772399902344,\n", - " 'referenceValueError': 1.52587890625e-05,\n", - " 'sphericalHarmonics': 0,\n", - " 'complexPacking': 0,\n", - " 'integerPointValues': 0,\n", - " 'additionalFlagPresent': 0,\n", - " 'orderOfSPD': 2,\n", - " 'boustrophedonic': 0,\n", - " 'hideThis': 0,\n", - " 'packingType': 'grid_simple',\n", - " 'bitsPerValue': 16,\n", - " 'constantFieldHalfByte': 8,\n", - " 'bitMapIndicator': 255,\n", - " 'numberOfCodedValues': 65160,\n", - " 'packingError': 0.00049591064453125,\n", - " 'unpackedError': 1.52587890625e-05,\n", - " 'maximum': 273.33799743652344,\n", - " 'minimum': 224.05772399902344,\n", - " 'average': 252.5098487718183,\n", - " 'standardDeviation': 13.372886915179645,\n", - " 'skewness': -0.2209591997514728,\n", - " 'kurtosis': -1.2738579926507174,\n", - " 'isConstant': 0.0,\n", - " 'numberOfMissing': 0,\n", - " 'dataLength': 16290,\n", - " 'changeDecimalPrecision': 0,\n", - " 'decimalPrecision': 0,\n", - " 'bitsPerValueAndRepack': 16,\n", - " 'setPackingType': 'grid_simple',\n", - " 'scaleValuesBy': 1.0,\n", - " 'offsetValuesBy': 0.0,\n", - " 'gridType': 'regular_ll',\n", - " 'getNumberOfValues': 65160,\n", - " 'padding_sec4_1': '',\n", - " 'md5Section4': '73097e43c94a4e2947a55799222424c2',\n", - " 'section5Length': 4,\n", - " '7777': '7777'},\n", - " 'ls': {'edition': 1,\n", - " 'centre': 'ecmf',\n", - " 'typeOfLevel': 'isobaricInhPa',\n", - " 'level': 500,\n", - " 'dataDate': 20070101,\n", - " 'stepRange': '0',\n", - " 'dataType': 'an',\n", - " 'shortName': 't',\n", - " 'packingType': 'grid_simple',\n", - " 'gridType': 'regular_ll'},\n", - " 'geography': {'bitmapPresent': 0,\n", - " 'Ni': 360,\n", - " 'Nj': 181,\n", - " 'latitudeOfFirstGridPointInDegrees': 90.0,\n", - " 'longitudeOfFirstGridPointInDegrees': 0.0,\n", - " 'latitudeOfLastGridPointInDegrees': -90.0,\n", - " 'longitudeOfLastGridPointInDegrees': 359.0,\n", - " 'iScansNegatively': 0,\n", - " 'jScansPositively': 0,\n", - " 'jPointsAreConsecutive': 0,\n", - " 'jDirectionIncrementInDegrees': 1.0,\n", - " 'iDirectionIncrementInDegrees': 1.0,\n", - " 'gridType': 'regular_ll'},\n", - " 'mars': {'domain': 'g',\n", - " 'levtype': 'pl',\n", - " 'levelist': 500,\n", - " 'date': 20070101,\n", - " 'time': 1200,\n", - " 'step': 0,\n", - " 'param': 't',\n", - " 'class': 'ea',\n", - " 'type': 'an',\n", - " 'stream': 'oper',\n", - " 'expver': '0001'},\n", - " 'parameter': {'centre': 'ecmf',\n", - " 'paramId': 130,\n", - " 'units': 'K',\n", - " 'name': 'Temperature',\n", - " 'shortName': 't'},\n", - " 'statistics': {'max': 273.33799743652344,\n", - " 'min': 224.05772399902344,\n", - " 'avg': 252.5098487718183,\n", - " 'sd': 13.372886915179645,\n", - " 'skew': -0.2209591997514728,\n", - " 'kurt': -1.2738579926507174,\n", - " 'const': 0.0},\n", - " 'time': {'dataDate': 20070101,\n", - " 'dataTime': 1200,\n", - " 'stepUnits': 1,\n", - " 'stepType': 'instant',\n", - " 'stepRange': '0',\n", - " 'startStep': 0,\n", - " 'endStep': 0,\n", - " 'validityDate': 20070101,\n", - " 'validityTime': 1200},\n", - " 'vertical': {'typeOfLevel': 'isobaricInhPa', 'level': 500}}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md.dump()" - ] - }, - { - "cell_type": "raw", - "id": "07b127e2-3401-44a7-9e83-5feb2e116b79", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "When we call :py:meth:`~data.readers.grib.metadata.GribMetadata.override` the new ecCodes handle is cloned and updated then added to the resulting object. The original metadata will not change." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "ac4195c6-9473-4db2-8747-38ddc5205a34", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "500\n", - "850\n" - ] - } - ], - "source": [ - "md1 = md.override(level=850)\n", - "print(md.get(\"level\"))\n", - "print(md1.get(\"level\"))" - ] - }, - { - "cell_type": "markdown", - "id": "b0dec726-be53-4fb0-b06d-7ca572e2655f", - "metadata": {}, - "source": [ - "Naturally, md1 still works when md is released:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "23d7a755-b46c-4b68-b5f1-e5c016e0f260", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "850" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "md = None\n", - "md1[\"level\"]" - ] - }, - { - "cell_type": "raw", - "id": "dc74c7bc-e6c7-4fad-970c-9471e03c1a39", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - ":py:meth:`~data.readers.grib.metadata.GribMetadata.override` also works with other Metadata objects:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "3ea5c463-e17e-4cce-9bd8-a10df6b58589", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "700\n", - "pt\n" - ] - } - ], - "source": [ - "raw_md = RawMetadata(level=700, shortName=\"pt\")\n", - "md2 = md1.override(raw_md)\n", - "print(md2.get(\"level\"))\n", - "print(md2.get(\"shortName\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6baea78-3c27-452e-81c7-262e481a5c58", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev_ecc", - "language": "python", - "name": "dev_ecc" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/cache.ipynb b/docs/examples/misc/cache.ipynb similarity index 99% rename from docs/examples/cache.ipynb rename to docs/examples/misc/cache.ipynb index e768788eb..abb48c376 100644 --- a/docs/examples/cache.ipynb +++ b/docs/examples/misc/cache.ipynb @@ -526,9 +526,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -540,7 +540,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/config.ipynb b/docs/examples/misc/config.ipynb similarity index 86% rename from docs/examples/config.ipynb rename to docs/examples/misc/config.ipynb index 9403af1a0..9145a056a 100644 --- a/docs/examples/config.ipynb +++ b/docs/examples/misc/config.ipynb @@ -111,16 +111,16 @@ " vertical-align: top;\n", " text-align: left !important;\n", "}\n", - "
NameValueDefault
cache-policy'off''off'
check-out-of-date-urlsTrueTrue
download-out-of-date-urlsFalseFalse
grib-field-policy'persistent''persistent'
grib-handle-cache-size11
grib-handle-policy'cache''cache'
maximum-cache-disk-usage'95%''95%'
maximum-cache-sizeNoneNone
number-of-download-threads55
reader-type-check-bytes6464
temporary-cache-directory-rootNoneNone
temporary-directory-rootNoneNone
url-download-timeout'30s''30s'
use-grib-metadata-cacheTrueTrue
use-message-position-index-cacheFalseFalse
use-standalone-mars-client-when-availableTrueTrue
user-cache-directory'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr''/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'
version'0.11.5.dev2+g384bbb0.d20241209'''
" + "
NameValueDefault
cache-policy'user''off'
check-out-of-date-urlsTrueTrue
download-out-of-date-urlsFalseFalse
grib-file-serialisation-policy'path''path'
grib-handle-cache-size11
grib-handle-policy'cache''cache'
maximum-cache-disk-usage'98%''98%'
maximum-cache-sizeNoneNone
number-of-download-threads55
reader-type-check-bytes6464
temporary-cache-directory-rootNoneNone
temporary-directory-rootNoneNone
url-download-timeout'30s''30s'
use-grib-metadata-cacheTrueTrue
use-message-position-index-cacheFalseFalse
use-standalone-mars-client-when-availableTrueTrue
user-cache-directory'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr''/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'
version'0.19.2.dev25+g34e3518fd.d20260302'''
" ], "text/plain": [ - "cache-policy: (off, off)\n", + "cache-policy: (user, off)\n", "check-out-of-date-urls: (True, True)\n", "download-out-of-date-urls: (False, False)\n", - "grib-field-policy: (persistent, persistent)\n", + "grib-file-serialisation-policy: (path, path)\n", "grib-handle-cache-size: (1, 1)\n", "grib-handle-policy: (cache, cache)\n", - "maximum-cache-disk-usage: (95%, 95%)\n", + "maximum-cache-disk-usage: (98%, 98%)\n", "maximum-cache-size: (None, None)\n", "number-of-download-threads: (5, 5)\n", "reader-type-check-bytes: (64, 64)\n", @@ -131,7 +131,7 @@ "use-message-position-index-cache: (False, False)\n", "use-standalone-mars-client-when-available: (True, True)\n", "user-cache-directory: (/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr, /var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr)\n", - "version: (0.11.5.dev2+g384bbb0.d20241209, )" + "version: (0.19.2.dev25+g34e3518fd.d20260302, )" ] }, "execution_count": 3, @@ -466,9 +466,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -480,7 +480,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/config_env_vars.ipynb b/docs/examples/misc/config_env_vars.ipynb similarity index 82% rename from docs/examples/config_env_vars.ipynb rename to docs/examples/misc/config_env_vars.ipynb index 32f4e839c..ff6b61d83 100644 --- a/docs/examples/config_env_vars.ipynb +++ b/docs/examples/misc/config_env_vars.ipynb @@ -214,7 +214,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/cgr/git/earthkit-data/src/earthkit/data/core/config.py:407: UserWarning: Config option 'url-download-timeout' is also set by environment variable 'EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT'.The environment variable takes precedence and its value is returned when calling get().\n", + "/Users/cgr/git/earthkit-data/src/earthkit/data/core/config.py:412: UserWarning: Config option 'url-download-timeout' is also set by environment variable 'EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT'.The environment variable takes precedence and its value is returned when calling get().\n", " warnings.warn(msg)\n" ] }, @@ -314,16 +314,16 @@ " vertical-align: top;\n", " text-align: left !important;\n", "}\n", - "
NameValueDefault
cache-policy'off''off'
check-out-of-date-urlsTrueTrue
download-out-of-date-urlsFalseFalse
grib-field-policy'persistent''persistent'
grib-handle-cache-size11
grib-handle-policy'cache''cache'
maximum-cache-disk-usage'95%''95%'
maximum-cache-sizeNoneNone
number-of-download-threads55
reader-type-check-bytes6464
temporary-cache-directory-rootNoneNone
temporary-directory-rootNoneNone
url-download-timeoutEARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT='26'
(10)
'30s'
use-grib-metadata-cacheTrueTrue
use-message-position-index-cacheFalseFalse
use-standalone-mars-client-when-availableTrueTrue
user-cache-directory'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr''/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'
version'0.11.5.dev2+g384bbb0.d20241209'''
" + "
NameValueDefault
cache-policy'user''off'
check-out-of-date-urlsTrueTrue
download-out-of-date-urlsFalseFalse
grib-file-serialisation-policy'path''path'
grib-handle-cache-size11
grib-handle-policy'cache''cache'
maximum-cache-disk-usage'98%''98%'
maximum-cache-sizeNoneNone
number-of-download-threads55
reader-type-check-bytes6464
temporary-cache-directory-rootNoneNone
temporary-directory-rootNoneNone
url-download-timeoutEARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT='26'
(10)
'30s'
use-grib-metadata-cacheTrueTrue
use-message-position-index-cacheFalseFalse
use-standalone-mars-client-when-availableTrueTrue
user-cache-directory'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr''/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'
version'0.19.2.dev25+g34e3518fd.d20260302'''
" ], "text/plain": [ - "cache-policy: (off, off)\n", + "cache-policy: (user, off)\n", "check-out-of-date-urls: (True, True)\n", "download-out-of-date-urls: (False, False)\n", - "grib-field-policy: (persistent, persistent)\n", + "grib-file-serialisation-policy: (path, path)\n", "grib-handle-cache-size: (1, 1)\n", "grib-handle-policy: (cache, cache)\n", - "maximum-cache-disk-usage: (95%, 95%)\n", + "maximum-cache-disk-usage: (98%, 98%)\n", "maximum-cache-size: (None, None)\n", "number-of-download-threads: (5, 5)\n", "reader-type-check-bytes: (64, 64)\n", @@ -334,7 +334,7 @@ "use-message-position-index-cache: (False, False)\n", "use-standalone-mars-client-when-available: (True, True)\n", "user-cache-directory: (/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr, /var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr)\n", - "version: (0.11.5.dev2+g384bbb0.d20241209, )" + "version: (0.19.2.dev25+g34e3518fd.d20260302, )" ] }, "execution_count": 8, @@ -363,9 +363,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -377,7 +377,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/demo_sources_plugin.ipynb b/docs/examples/misc/demo_sources_plugin.ipynb similarity index 100% rename from docs/examples/demo_sources_plugin.ipynb rename to docs/examples/misc/demo_sources_plugin.ipynb diff --git a/docs/examples/misc/index.rst b/docs/examples/misc/index.rst new file mode 100644 index 000000000..0c7781aff --- /dev/null +++ b/docs/examples/misc/index.rst @@ -0,0 +1,14 @@ +.. _misc_examples: + +Miscellaneous ++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + config.ipynb + config_env_vars.ipynb + cache.ipynb + projection.ipynb + demo_sources_plugin.ipynb diff --git a/docs/examples/projection.ipynb b/docs/examples/misc/projection.ipynb similarity index 95% rename from docs/examples/projection.ipynb rename to docs/examples/misc/projection.ipynb index 96137c2ea..bd58938cb 100644 --- a/docs/examples/projection.ipynb +++ b/docs/examples/misc/projection.ipynb @@ -156,7 +156,7 @@ " \n", " \n", " \n", - " 2026-03-11T12:03:08.267832\n", + " 2026-03-13T19:27:07.321321\n", " image/svg+xml\n", " \n", " \n", @@ -258,7 +258,7 @@ "L 86.328238 171.095907 \n", "L 86.320854 171.105652 \n", "L 86.574058 171.16764 \n", - "\" clip-path=\"url(#pef00259059)\" style=\"fill: none; stroke: #000000\"/>\n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", " \n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", " \n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p5acf8a2bc6)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", "\n", - "
<cartopy.crs.LambertAzimuthalEqualArea object at 0x107644440>
" + "
<cartopy.crs.LambertAzimuthalEqualArea object at 0x111be02f0>
" ], "text/plain": [ "\n", @@ -6207,7 +6207,7 @@ " \n", " \n", " \n", - " 2026-03-11T12:03:08.657793\n", + " 2026-03-13T19:27:07.731876\n", " image/svg+xml\n", " \n", " \n", @@ -6252,7 +6252,7 @@ "L 20.353759 138.1198 \n", "L 20.323382 138.074224 \n", "L 19.822506 137.861642 \n", - "\" clip-path=\"url(#p12f9a01820)\" style=\"fill: none; stroke: #000000\"/>\n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #000000\"/>\n", " \n", " \n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", " \n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", + "\" clip-path=\"url(#p87994ed6f1)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8\"/>\n", " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", "\n", - "
<cartopy.crs.PlateCarree object at 0x1497ba750>
" + "
<cartopy.crs.PlateCarree object at 0x1321c6550>
" ], "text/plain": [ "\n", diff --git a/docs/examples/netcdf/index.rst b/docs/examples/netcdf/index.rst new file mode 100644 index 000000000..a4bc9a9e1 --- /dev/null +++ b/docs/examples/netcdf/index.rst @@ -0,0 +1,12 @@ +.. _netcdf_examples: + + +NetCDF +++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + netcdf.ipynb + netcdf_fieldlist.ipynb diff --git a/docs/examples/netcdf.ipynb b/docs/examples/netcdf/netcdf.ipynb similarity index 52% rename from docs/examples/netcdf.ipynb rename to docs/examples/netcdf/netcdf.ipynb index 6bff72d25..778a8979e 100644 --- a/docs/examples/netcdf.ipynb +++ b/docs/examples/netcdf/netcdf.ipynb @@ -27,75 +27,41 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], - "source": [ - "fs = ekd.from_source(\"file\", \"test.nc\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "earthkit.data.readers.netcdf.NetCDFFieldListReader" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "type(fs)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "

NetCDF file
Path: test.nc size: 6.4 KiB
Available types: xarray, pandas, fieldlist, numpy, array

" + ], "text/plain": [ - "2" + "" ] }, - "execution_count": 4, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "len(fs)" + "d = ekd.from_source(\"file\", \"test.nc\")\n", + "d" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "NetCDFField(t2m,)\n", - "NetCDFField(msl,)\n" + "/Users/cgr/git/earthkit-data/src/earthkit/data/readers/netcdf/reader.py:57: FutureWarning: In a future version, xarray will not decode the variable 'step' into a timedelta64 dtype based on the presence of a timedelta-like 'units' attribute by default. Instead it will rely on the presence of a timedelta64 'dtype' attribute, which is now xarray's default way of encoding timedelta64 values.\n", + "To continue decoding into a timedelta64 dtype, either set `decode_timedelta=True` when opening this dataset, or add the attribute `dtype='timedelta64[ns]'` to this variable on disk.\n", + "To opt-in to future behavior, set `decode_timedelta=False`.\n", + " return xr.open_mfdataset(\n" ] - } - ], - "source": [ - "for f in fs:\n", - " print(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ + }, { "data": { "text/html": [ @@ -114,39 +80,86 @@ "\n", "\n", "\n", - "
<xarray.Dataset> Size: 2kB\n",
-       "Dimensions:     (longitude: 19, latitude: 11)\n",
+       "Dimensions:     (latitude: 11, longitude: 19)\n",
        "Coordinates:\n",
-       "  * longitude   (longitude) float64 152B -27.0 -23.0 -19.0 ... 37.0 41.0 45.0\n",
        "  * latitude    (latitude) float64 88B 73.0 69.0 65.0 61.0 ... 41.0 37.0 33.0\n",
+       "  * longitude   (longitude) float64 152B -27.0 -23.0 -19.0 ... 37.0 41.0 45.0\n",
        "    number      int32 4B ...\n",
        "    time        datetime64[ns] 8B ...\n",
        "    step        timedelta64[ns] 8B ...\n",
@@ -488,8 +563,8 @@
        "    GRIB_subCentre:          0\n",
        "    Conventions:             CF-1.7\n",
        "    institution:             European Centre for Medium-Range Weather Forecasts\n",
-       "    history:                 GRIB to CDM+CF via cfgrib-0.9.5/ecCodes-2.17.0 w...
" ], "text/plain": [ " Size: 2kB\n", diff --git a/docs/examples/ecmwf_open_data.ipynb b/docs/examples/source/ecmwf_open_data.ipynb similarity index 100% rename from docs/examples/ecmwf_open_data.ipynb rename to docs/examples/source/ecmwf_open_data.ipynb diff --git a/docs/examples/fdb.ipynb b/docs/examples/source/fdb.ipynb similarity index 99% rename from docs/examples/fdb.ipynb rename to docs/examples/source/fdb.ipynb index 7b24459d6..748cada80 100644 --- a/docs/examples/fdb.ipynb +++ b/docs/examples/source/fdb.ipynb @@ -145,7 +145,7 @@ } ], "source": [ - "ds = earthkit.data.from_source(\"fdb\", request=request)\n", + "ds = earthkit.data.from_source(\"fdb\", request=request).to_fieldlist()\n", "for f in ds:\n", " print(f)" ] @@ -226,7 +226,7 @@ } ], "source": [ - "ds = earthkit.data.from_source(\"fdb\", request=request)\n", + "ds = earthkit.data.from_source(\"fdb\", request=request).to_fieldlist()\n", "for f in ds.group_by(\"time\"):\n", " print(f\"len={len(f)} {f.metadata(('param', 'level'))}\")" ] @@ -277,7 +277,7 @@ } ], "source": [ - "ds = earthkit.data.from_source(\"fdb\", request=request)\n", + "ds = earthkit.data.from_source(\"fdb\", request=request).to_fieldlist()\n", "for f in ds.batched(2):\n", " print(f\"len={len(f)} {f.metadata(('param', 'level'))}\")" ] @@ -302,7 +302,7 @@ "tags": [] }, "source": [ - "We can load the whole stream into memory by using ``read_all=True`` in :ref:`from_source() `. The resulting object will be a FieldList." + "We can load the whole stream into memory by using ``read_all=True`` in ``to_fieldlist()``. The resulting object will be a FieldList." ] }, { @@ -318,7 +318,7 @@ }, "outputs": [], "source": [ - "ds = earthkit.data.from_source(\"fdb\", request=request, read_all=True)" + "ds = earthkit.data.from_source(\"fdb\", request=request).to_fieldlist(read_all=True)" ] }, { @@ -1048,7 +1048,7 @@ }, "outputs": [], "source": [ - "ds = earthkit.data.from_source(\"fdb\", request=request, stream=False)" + "ds = earthkit.data.from_source(\"fdb\", request=request, stream=False).to_fieldlist()" ] }, { diff --git a/docs/examples/file_parts.ipynb b/docs/examples/source/file_parts.ipynb similarity index 100% rename from docs/examples/file_parts.ipynb rename to docs/examples/source/file_parts.ipynb diff --git a/docs/examples/file_stream.ipynb b/docs/examples/source/file_stream.ipynb similarity index 92% rename from docs/examples/file_stream.ipynb rename to docs/examples/source/file_stream.ipynb index 06549ed29..e3c7103f6 100644 --- a/docs/examples/file_stream.ipynb +++ b/docs/examples/source/file_stream.ipynb @@ -75,9 +75,42 @@ }, "tags": [] }, + "outputs": [ + { + "data": { + "text/html": [ + "

Stream of fields
Available types: fieldlist

" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_in = ekd.from_source(\"file\", \"test6.grib\", stream=True)\n", + "ds_in" + ] + }, + { + "cell_type": "markdown", + "id": "29ec8353-a5fe-401c-a517-f4a324828c72", + "metadata": {}, + "source": [ + "To access the stream data we need to convert the data object into a stream fieldlist." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "899555f6-8e8a-4c37-810d-dc85b9267d62", + "metadata": {}, "outputs": [], "source": [ - "ds = ekd.from_source(\"file\", \"test6.grib\", stream=True).to_fieldlist()" + "ds = ds_in.to_fieldlist()" ] }, { @@ -97,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "animated-prayer", "metadata": { "editable": true, @@ -142,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "de7520fa-abad-40dd-97c5-bb5868859e2b", "metadata": { "editable": true, @@ -158,7 +191,7 @@ "1" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -198,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "placed-blues", "metadata": { "editable": true, @@ -243,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "bf94a190-ec0e-4172-8e75-6518e48f50a4", "metadata": { "editable": true, @@ -295,7 +328,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "8e1be478-6eb6-4732-bb96-9d6fa942c20d", "metadata": { "editable": true, @@ -352,7 +385,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "simple-london", "metadata": { "editable": true, @@ -368,7 +401,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "meaning-oxide", "metadata": {}, "outputs": [ @@ -378,7 +411,7 @@ "6" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -389,7 +422,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "copyrighted-walnut", "metadata": { "editable": true, @@ -519,7 +552,7 @@ "5 850 pressure 0 regular_ll " ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -530,7 +563,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "static-reasoning", "metadata": {}, "outputs": [ @@ -602,7 +635,7 @@ "1 850 pressure 0 regular_ll " ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -614,7 +647,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "spanish-wagon", "metadata": { "editable": true, @@ -1115,7 +1148,7 @@ " t (level, latitude, longitude) float64 1kB ...\n", "Attributes:\n", " Conventions: CF-1.8\n", - " institution: ECMWF" + " institution: ECMWF" ], "text/plain": [ " Size: 2kB\n", @@ -1131,7 +1164,7 @@ " institution: ECMWF" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } diff --git a/docs/examples/files.ipynb b/docs/examples/source/files.ipynb similarity index 87% rename from docs/examples/files.ipynb rename to docs/examples/source/files.ipynb index 5ba6945f5..06d4e624f 100644 --- a/docs/examples/files.ipynb +++ b/docs/examples/source/files.ipynb @@ -73,7 +73,7 @@ "tags": [] }, "source": [ - "Files can be read by :ref:`from_source ` as a \"file\" source. The resulting object depends on the file type and chosen to best represent the the input data." + "Files can be read by :ref:`from_source ` as a \"file\" source. The resulting object depends privides some deatils about the data and to types it can converted to for further work." ] }, { @@ -91,69 +91,10 @@ { "data": { "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
02t2020-05-13 12:00:002020-05-13 12:00:000 days0surface0regular_ll
1msl2020-05-13 12:00:002020-05-13 12:00:000 days0surface0regular_ll
\n", - "
" + "

GRIB file
Path: test.grib size: 1 KiB
Available types: fieldlist, pandas, xarray, numpy, array

" ], "text/plain": [ - " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", - "0 2t 2020-05-13 12:00:00 2020-05-13 12:00:00 0 days \n", - "1 msl 2020-05-13 12:00:00 2020-05-13 12:00:00 0 days \n", - "\n", - " vertical.level vertical.level_type ensemble.member geography.grid_type \n", - "0 0 surface 0 regular_ll \n", - "1 0 surface 0 regular_ll " + "GribData(path={self._reader.path})" ] }, "execution_count": 3, @@ -163,13 +104,23 @@ ], "source": [ "# grib data\n", - "ds = ekd.from_source(\"file\", \"test.grib\")\n", - "ds.to_fieldlist().ls()" + "ds_in = ekd.from_source(\"file\", \"test.grib\")\n", + "ds_in" ] }, { "cell_type": "code", "execution_count": 4, + "id": "6661e62d-a33b-498a-8373-55a018cd024b", + "metadata": {}, + "outputs": [], + "source": [ + "ds = ds_in.to_fieldlist().ls()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "3d0be6a1-6bac-4e74-b461-ec5c58b7112d", "metadata": { "editable": true, @@ -178,6 +129,32 @@ }, "tags": [] }, + "outputs": [ + { + "data": { + "text/html": [ + "

NetCDF file
Path: test.nc size: 6.4 KiB
Available types: xarray, pandas, fieldlist, numpy, array

" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# netcdf data\n", + "ds_in = ekd.from_source(\"file\", \"test.nc\")\n", + "ds_in" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4c224418-7962-4de1-912f-8c576bbba888", + "metadata": {}, "outputs": [ { "name": "stderr", @@ -690,8 +667,8 @@ " GRIB_subCentre: 0\n", " Conventions: CF-1.7\n", " institution: European Centre for Medium-Range Weather Forecasts\n", - " history: GRIB to CDM+CF via cfgrib-0.9.5/ecCodes-2.17.0 w..." - ], - "text/plain": [ - " Size: 176kB\n", - "Dimensions: (forecast_reference_time: 4, step: 2, level: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * level (level) int64 16B 500 700\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables:\n", - " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import earthkit.data as ekd\n", - "\n", - "ds_fl = ekd.from_source(\"sample\", \"pl.grib\")\n", - "ds_xr = ds_fl.to_xarray()\n", - "ds_xr" - ] - }, - { - "cell_type": "raw", - "id": "28af4ff2-aaf3-4952-855d-3b91588e2de5", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "By default, ``add_earthkit_attrs=True`` in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` and some special earthkit attributes are added to the dataset. This is needed for the Xarray to GRIB conversion. For this reason, if the Xarray is modified we must ensure the variable attributes are copied to the new Xarray dataset. By default, variable attributes are not kept in Xarray computations so we need to set the global Xarray ``keep_attrs`` option to enable it." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "b98402ad-8e81-42af-9426-c8c59d0d1a45", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# ensure earthkit attributes are set\n", - "import xarray as xr \n", - "xr.set_options(keep_attrs=True)\n", - "\n", - "# modify values\n", - "ds_xr += 1" - ] - }, - { - "cell_type": "markdown", - "id": "40c77039-21e0-48f4-a809-9530672da724", - "metadata": {}, - "source": [ - "#### Using to_target()" - ] - }, - { - "cell_type": "raw", - "id": "d6c82521-859e-4bc9-ad3e-686071af88e8", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "It is possible to directly write the Xarray dataset created with the earthkit engine into a GRIB file with :func:`to_target`. This is a memory efficient way to write GRIB to disk since only one field is loaded into memory at a time. We can call :func:`to_target` either on the ``earthkit`` accessor or as a top level function." - ] - }, - { - "cell_type": "markdown", - "id": "fcf81205-624e-4d4c-ad86-0595ff3e588f", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "First, we write a datarray into a GRIB file." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "68d4ec97-2429-44f8-8f80-5ffd58639222", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "GribField(t,500,20240603,0,0,0)" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# option1: writing to GRIB file using the accessor\n", - "ds_xr[\"t\"].earthkit.to_target(\"file\", \"_from_xr_1.grib\")\n", - "\n", - "# option2: writing to GRIB file using the top level function\n", - "ekd.to_target(\"file\", \"_from_xr_1a.grib\", data=ds_xr[\"t\"])\n", - "\n", - "# check the results\n", - "ds_tmp1 = ekd.from_source(\"file\", \"_from_xr_1.grib\")\n", - "ds_tmp1[0]" - ] - }, - { - "cell_type": "markdown", - "id": "d31c0a87-ce0f-4365-a9d4-f9fb88e67eaa", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Next, we write the whole dataset into a GRIB file." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "ced38f9d-20c6-43d7-9e91-32b7bff93a6a", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmfrisobaricInhPa5002024060300fc0regular_ll
1ecmfrisobaricInhPa7002024060300fc0regular_ll
2ecmfrisobaricInhPa5002024060306fc0regular_ll
3ecmfrisobaricInhPa7002024060306fc0regular_ll
4ecmfrisobaricInhPa5002024060312000fc0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf r isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf r isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf r isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf r isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf r isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# option1: writing to GRIB file using the accessor\n", - "ds_xr.earthkit.to_target(\"file\", \"_from_xr_2.grib\")\n", - "\n", - "# option2: writing to GRIB file using the top level function\n", - "ekd.to_target(\"file\", \"_from_xr_2a.grib\", data=ds_xr)\n", - "\n", - "# check the results\n", - "ds_tmp2 = ekd.from_source(\"file\", \"_from_xr_2.grib\")\n", - "ds_tmp2.head()" - ] - }, - { - "cell_type": "markdown", - "id": "c75cbad6-a383-4bf2-add9-98a6c313aada", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We check if the computation results were correctly written to the generated GRIB data." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "0c0bafa4-e3ca-4361-b28c-851b8447f323", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "250.22500610351562\n", - "251.22500610351562\n" - ] - } - ], - "source": [ - "# original GRIB data\n", - "print(ds_fl.sel(param=\"t\", step=0, level=500)[0].values[0])\n", - "# GRIB data converted from the modified xarray object\n", - "print(ds_tmp1.sel(param=\"t\", step=0, level=500)[0].values[0])" - ] - }, - { - "cell_type": "markdown", - "id": "be7471ec-bb6c-4873-bd32-babb4b4fc2ba", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Using to_fieldlist()" - ] - }, - { - "cell_type": "raw", - "id": "30ad2d99-db56-4302-99cf-277602202459", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can also convert the Xarray dataset into a GRIB fieldlist by using :py:meth:`~data.xr_engine.engine.XarrayEarthkit.to_fieldlist` on the ``earthkit`` accessor of the Xarray object. Please note that this will generate a fieldlist entirely stored in memory." - ] - }, - { - "cell_type": "markdown", - "id": "07ee1067-6476-450b-bc12-d90b1dc2fc05", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "First, we convert a dataarray to a GRIB fieldlist." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "3784d2d3-8254-48bf-8ee2-9aa2b5d69686", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmftisobaricInhPa5002024060300fc0regular_ll
1ecmftisobaricInhPa7002024060300fc0regular_ll
2ecmftisobaricInhPa5002024060306fc0regular_ll
3ecmftisobaricInhPa7002024060306fc0regular_ll
4ecmftisobaricInhPa5002024060312000fc0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf t isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf t isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf t isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf t isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_fl1 = ds_xr[\"t\"].earthkit.to_fieldlist()\n", - "ds_fl1.head()" - ] - }, - { - "cell_type": "markdown", - "id": "b81cc6de-0b91-442b-9df3-4f3c332aa09c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "Next, we convert back the whole dataset into a GRIB fieldlist." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "b39c4d26-dd9f-4a76-9559-6fb8b6676698", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
centreshortNametypeOfLevelleveldataDatedataTimestepRangedataTypenumbergridType
0ecmfrisobaricInhPa5002024060300fc0regular_ll
1ecmfrisobaricInhPa7002024060300fc0regular_ll
2ecmfrisobaricInhPa5002024060306fc0regular_ll
3ecmfrisobaricInhPa7002024060306fc0regular_ll
4ecmfrisobaricInhPa5002024060312000fc0regular_ll
\n", - "
" - ], - "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf r isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf r isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf r isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf r isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf r isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_fl2 = ds_xr.earthkit.to_fieldlist()\n", - "ds_fl2.head()" - ] - }, - { - "cell_type": "raw", - "id": "d67d36c7-e0e4-41d5-b0b6-e8b8743f3d10", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The generated GRIB fieldlist can be saved to disk using the :func:`to_target` method." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "36423326-cf54-494b-8932-72d986b908bf", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "GribField(t,500,20240603,0,0,0)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "out_name = \"_from_xr_3.grib\"\n", - "ds_fl1.to_target(\"file\", out_name)\n", - "# read back and check the saved GRIB\n", - "ds_tmp = ekd.from_source(\"file\", out_name)\n", - "ds_tmp[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c7a3570-2514-42f9-8b7c-0a087fcc9205", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/xarray_engine_variable_key.ipynb b/docs/examples/xarray_engine_variable_key.ipynb deleted file mode 100644 index 1755d28e8..000000000 --- a/docs/examples/xarray_engine_variable_key.ipynb +++ /dev/null @@ -1,2924 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "bc9477f3-1aa0-4539-acef-0bd2b43191ba", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "## Xarray engine: variable key" - ] - }, - { - "cell_type": "raw", - "id": "236700a8-2212-4675-bd0d-d91159219a6a", - "metadata": { - "editable": true, - "raw_mimetype": "text/restructuredtext", - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The ``variable_key`` option in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` controls what metadata key should be used to form the DataArray variables. By default it is \"param\", which is an alias to \"shortName\" for GRIB in earthkit-data.\n", - "\n", - "Please note it is also possible to generate an Xarray with a single dataarray containing all the parameters from a GRIB fieldlist. See e.g. the :ref:`/examples/xarray_engine_mono_variable.ipynb` notebook for details." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "08b75c56-0b2f-4cc6-9637-b28ad2aa4455", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4b267deb657948f6a4992cde6a710f41", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "pl.grib: 0%| | 0.00/48.8k [00:00\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 176kB\n",
-       "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
-       "                              latitude: 19, longitude: 36)\n",
-       "Coordinates:\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * level                    (level) int64 16B 500 700\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "Data variables:\n",
-       "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
-       "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    levtype:      pl\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 176kB\n", - "Dimensions: (forecast_reference_time: 4, step: 2, level: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * level (level) int64 16B 500 700\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables:\n", - " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import earthkit.data as ekd\n", - "ds_fl = ekd.from_source(\"sample\", \"pl.grib\")\n", - "ds = ds_fl.to_xarray()\n", - "ds" - ] - }, - { - "cell_type": "markdown", - "id": "7c2697d5-882e-4b47-977d-f45a91c222c2", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### The param_level key" - ] - }, - { - "cell_type": "markdown", - "id": "08015f0b-65de-44f0-beda-c75390615352", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "The built-in \"param_level\" metadata key combines the values of the \"param\" and \"level\" metadata keys as a str. We can use it as ``variable_key`` to build our Xarray." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4898ceb3-4657-4397-b1d8-3bc1110b86eb", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 176kB\n",
-       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
-       "                              latitude: 19, longitude: 36)\n",
-       "Coordinates:\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "Data variables:\n",
-       "    r500                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r700                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    t500                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    t700                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    levtype:      pl\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 176kB\n", - "Dimensions: (forecast_reference_time: 4, step: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables:\n", - " r500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r700 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " t500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " t700 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds = ds_fl.to_xarray(variable_key=\"param_level\")\n", - "ds" - ] - }, - { - "cell_type": "markdown", - "id": "0921516f-74b2-417a-a85a-378525090348", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "This technique can come in handy when there are parameters with different level types in the input data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "f55fe598-7f80-4e9a-87b3-43131f78f2e1", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d314ccebbb2d4072b01a5774f4bfd637", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "mixed_pl_sfc.grib: 0%| | 0.00/390k [00:00\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 1MB\n",
-       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
-       "                              latitude: 19, longitude: 36)\n",
-       "Coordinates:\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "Data variables: (12/32)\n",
-       "    2t0                      (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    msl0                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r1000                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r300                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r400                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r500                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    ...                       ...\n",
-       "    z1000                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z300                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z400                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z500                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z700                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z850                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 1MB\n", - "Dimensions: (forecast_reference_time: 4, step: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables: (12/32)\n", - " 2t0 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " msl0 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r1000 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r300 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r400 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " ... ...\n", - " z1000 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z300 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z400 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z700 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z850 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_sfc.grib\")\n", - "ds = ds_fl.to_xarray(variable_key=\"param_level\")\n", - "ds" - ] - }, - { - "cell_type": "markdown", - "id": "1939ee80-8c1e-4498-8a80-d5e6ac65f8e7", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "#### Using remapping" - ] - }, - { - "cell_type": "markdown", - "id": "7e22e364-2518-4460-ab35-64230d49a703", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "We can take it one step further and define a metadata key that combines the param, the level and the level type into a single key. We can achieve it by using the ``remapping`` option." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "e1bc4e80-2efc-460c-96b0-26553bed7591", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2fb6d5cb8cf44e839b0ec741cb1f041b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "mixed_pl_sfc.grib: 0%| | 0.00/390k [00:00\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 1MB\n",
-       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
-       "                              latitude: 19, longitude: 36)\n",
-       "Coordinates:\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "Data variables: (12/32)\n",
-       "    2t_sfc                   (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    msl_sfc                  (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r_1000_pl                (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r_300_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r_400_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    r_500_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    ...                       ...\n",
-       "    z_1000_pl                (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z_300_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z_400_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z_500_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z_700_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    z_850_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 1MB\n", - "Dimensions: (forecast_reference_time: 4, step: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables: (12/32)\n", - " 2t_sfc (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " msl_sfc (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r_1000_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r_300_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r_400_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " r_500_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " ... ...\n", - " z_1000_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z_300_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z_400_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z_500_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z_700_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " z_850_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_sfc.grib\")\n", - "ds = ds_fl.to_xarray(variable_key=\"p_l_t\", remapping={\"p_l_t\": \"{param}_{levelist}_{levtype}\"})\n", - "ds" - ] - }, - { - "cell_type": "markdown", - "id": "374ea92f-5b3f-4134-adfb-1c93dc1258f7", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "This technique is particularly useful when the same parameter is available on multiple level types in the input data. In this case using \"param_level\" does not result in a full hypercube, however the same ``remapping`` that we used above does." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "39cbb360-43c4-416e-956b-8b6cfadda26c", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b69825ffceb9401da6c4b0de9c92ed99", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "mixed_pl_ml.grib: 0%| | 0.00/176k [00:00\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 351kB\n",
-       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
-       "                              latitude: 19, longitude: 36)\n",
-       "Coordinates:\n",
-       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
-       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
-       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
-       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
-       "Data variables:\n",
-       "    t_137_ml                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    t_500_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    t_700_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    t_90_ml                  (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    u_137_ml                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    u_500_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    u_700_pl                 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "    u_90_ml                  (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
-       "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
" - ], - "text/plain": [ - " Size: 351kB\n", - "Dimensions: (forecast_reference_time: 4, step: 2,\n", - " latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", - " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", - " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", - "Data variables:\n", - " t_137_ml (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " t_500_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " t_700_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " t_90_ml (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " u_137_ml (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " u_500_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " u_700_pl (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - " u_90_ml (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_ml.grib\")\n", - "ds = ds_fl.to_xarray(variable_key=\"p_l_t\", remapping={\"p_l_t\": \"{param}_{levelist}_{levtype}\"})\n", - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4be3d1c2-873b-42a7-9a92-468d38ee64e2", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev", - "language": "python", - "name": "dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/xr_engine/index.rst b/docs/examples/xr_engine/index.rst new file mode 100644 index 000000000..169f4f773 --- /dev/null +++ b/docs/examples/xr_engine/index.rst @@ -0,0 +1,29 @@ +.. _xr_engine_examples: + +Xarray engine ++++++++++++++ + +.. toctree:: + :maxdepth: 1 + :glob: + + xarray_engine_overview.ipynb + xarray_engine_temporal.ipynb + xarray_engine_step_ranges.ipynb + xarray_engine_seasonal.ipynb + xarray_engine_level.ipynb + xarray_engine_ensemble.ipynb + xarray_engine_variable_key.ipynb + xarray_engine_mono_variable.ipynb + xarray_engine_mono_variable_remapping.ipynb + xarray_engine_field_dims.ipynb + xarray_engine_to_grib.ipynb + xarray_engine_split.ipynb + xarray_engine_squeeze.ipynb + xarray_engine_dims_as_attrs.ipynb + xarray_engine_extra_dims.ipynb + xarray_engine_remapping.ipynb + xarray_engine_holes.ipynb + xarray_engine_chunks.ipynb + xarray_engine_chunks_on_dask_cluster.ipynb + xarray_cupy.ipynb diff --git a/docs/examples/xarray_engine_chunks.ipynb b/docs/examples/xr_engine/xarray_engine_chunks.ipynb similarity index 100% rename from docs/examples/xarray_engine_chunks.ipynb rename to docs/examples/xr_engine/xarray_engine_chunks.ipynb diff --git a/docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb b/docs/examples/xr_engine/xarray_engine_chunks_on_dask_cluster.ipynb similarity index 100% rename from docs/examples/xarray_engine_chunks_on_dask_cluster.ipynb rename to docs/examples/xr_engine/xarray_engine_chunks_on_dask_cluster.ipynb diff --git a/docs/examples/xarray_engine_dims_as_attrs.ipynb b/docs/examples/xr_engine/xarray_engine_dims_as_attrs.ipynb similarity index 100% rename from docs/examples/xarray_engine_dims_as_attrs.ipynb rename to docs/examples/xr_engine/xarray_engine_dims_as_attrs.ipynb diff --git a/docs/examples/xarray_engine_ensemble.ipynb b/docs/examples/xr_engine/xarray_engine_ensemble.ipynb similarity index 100% rename from docs/examples/xarray_engine_ensemble.ipynb rename to docs/examples/xr_engine/xarray_engine_ensemble.ipynb diff --git a/docs/examples/xarray_engine_extra_dims.ipynb b/docs/examples/xr_engine/xarray_engine_extra_dims.ipynb similarity index 100% rename from docs/examples/xarray_engine_extra_dims.ipynb rename to docs/examples/xr_engine/xarray_engine_extra_dims.ipynb diff --git a/docs/examples/xarray_engine_field_dims.ipynb b/docs/examples/xr_engine/xarray_engine_field_dims.ipynb similarity index 100% rename from docs/examples/xarray_engine_field_dims.ipynb rename to docs/examples/xr_engine/xarray_engine_field_dims.ipynb diff --git a/docs/examples/xarray_engine_holes.ipynb b/docs/examples/xr_engine/xarray_engine_holes.ipynb similarity index 100% rename from docs/examples/xarray_engine_holes.ipynb rename to docs/examples/xr_engine/xarray_engine_holes.ipynb diff --git a/docs/examples/xarray_engine_level.ipynb b/docs/examples/xr_engine/xarray_engine_level.ipynb similarity index 100% rename from docs/examples/xarray_engine_level.ipynb rename to docs/examples/xr_engine/xarray_engine_level.ipynb diff --git a/docs/examples/xarray_engine_mono_variable.ipynb b/docs/examples/xr_engine/xarray_engine_mono_variable.ipynb similarity index 100% rename from docs/examples/xarray_engine_mono_variable.ipynb rename to docs/examples/xr_engine/xarray_engine_mono_variable.ipynb diff --git a/docs/examples/xarray_engine_mono_variable_remapping.ipynb b/docs/examples/xr_engine/xarray_engine_mono_variable_remapping.ipynb similarity index 100% rename from docs/examples/xarray_engine_mono_variable_remapping.ipynb rename to docs/examples/xr_engine/xarray_engine_mono_variable_remapping.ipynb diff --git a/docs/examples/xarray_engine_overview.ipynb b/docs/examples/xr_engine/xarray_engine_overview.ipynb similarity index 57% rename from docs/examples/xarray_engine_overview.ipynb rename to docs/examples/xr_engine/xarray_engine_overview.ipynb index 9b51f2fe9..6e892b853 100644 --- a/docs/examples/xarray_engine_overview.ipynb +++ b/docs/examples/xr_engine/xarray_engine_overview.ipynb @@ -53,25 +53,10 @@ }, "tags": [] }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8b2a598b3f264e2aa75a0cddab1650d2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "pl.grib: 0%| | 0.00/48.8k [00:00\n", "\n", "\n", - "
<xarray.Dataset> Size: 176kB\n",
        "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
        "                              latitude: 19, longitude: 36)\n",
@@ -501,30 +594,13 @@
        "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
        "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
        "Attributes:\n",
-       "    class:        od\n",
-       "    stream:       oper\n",
-       "    levtype:      pl\n",
-       "    type:         fc\n",
-       "    expver:       0001\n",
-       "    date:         20240603\n",
-       "    time:         0\n",
-       "    domain:       g\n",
-       "    number:       0\n",
        "    Conventions:  CF-1.8\n",
-       "    institution:  ECMWF
    • r
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", @@ -540,15 +616,6 @@ " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -690,81 +757,69 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 4\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 12:00:00\n", + " 2024-06-03 12:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 1200\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -773,19 +828,19 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf r isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf r isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf r isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf r isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf r isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 r 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " ] }, "execution_count": 4, @@ -814,7 +869,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "id": "5ad32a3e-2f48-49f5-b207-15e89c397fba", "metadata": { "editable": true, @@ -827,17 +882,17 @@ { "data": { "text/plain": [ - "(254.25649845948692, 255.25649845948692)" + "(np.float64(254.25649845948692), np.float64(255.25649845948692))" ] }, - "execution_count": 8, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "m_0 = ds_fl.sel(param=\"t\", step=6, level=500)[0].values.mean() \n", - "m_1 = ds_fl1.sel(param=\"t\", step=6, level=500)[0].values.mean()\n", + "m_0 = ds_fl.sel({\"parameter.variable\": \"t\", \"time.step\": 6, \"vertical.level\": 500})[0].values.mean() \n", + "m_1 = ds_fl1.sel({\"parameter.variable\": \"t\", \"time.step\": 6, \"vertical.level\": 500})[0].values.mean()\n", "m_0, m_1" ] }, @@ -903,81 +958,69 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 4\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 12:00:00\n", + " 2024-06-03 12:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 1200\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -986,19 +1029,19 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf r isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf r isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf r isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf r isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf r isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 r 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " ] }, "execution_count": 6, @@ -1008,7 +1051,7 @@ ], "source": [ "ds_fl1.to_target(\"file\", \"_from_xr_1.grib\")\n", - "ekd.from_source(\"file\", \"_from_xr_1.grib\").head()" + "ekd.from_source(\"file\", \"_from_xr_1.grib\").to_fieldlist().head()" ] }, { @@ -1023,7 +1066,7 @@ "tags": [] }, "source": [ - "It is also possible to directly write the Xarray into a GRIB file when calling :py:meth:`~data.xr_engine.engine.XarrayEarthkit.to_grib` on the ``earthkit`` accessor. This will be a more memory efficient way to write the data to disk than generating a fieldlist first." + "It is also possible to directly write the Xarray into a GRIB file when calling ``to_target()`` on the ``earthkit`` accessor. This will be a more memory efficient way to write the data to disk than generating a fieldlist first." ] }, { @@ -1059,81 +1102,69 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 00:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03 00:00:00\n", + " 0 days 06:00:00\n", " 700\n", - " 20240603\n", - " 0\n", - " 6\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 4\n", - " ecmf\n", " r\n", - " isobaricInhPa\n", + " 2024-06-03 12:00:00\n", + " 2024-06-03 12:00:00\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 1200\n", - " 0\n", - " fc\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", @@ -1142,19 +1173,19 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf r isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf r isobaricInhPa 700 20240603 0 0 \n", - "2 ecmf r isobaricInhPa 500 20240603 0 6 \n", - "3 ecmf r isobaricInhPa 700 20240603 0 6 \n", - "4 ecmf r isobaricInhPa 500 20240603 1200 0 \n", - "\n", - " dataType number gridType \n", - "0 fc 0 regular_ll \n", - "1 fc 0 regular_ll \n", - "2 fc 0 regular_ll \n", - "3 fc 0 regular_ll \n", - "4 fc 0 regular_ll " + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 r 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " ] }, "execution_count": 7, @@ -1163,8 +1194,8 @@ } ], "source": [ - "ds.earthkit.to_grib(\"_from_xr_2.grib\")\n", - "ekd.from_source(\"file\", \"_from_xr_2.grib\").head()" + "ds.earthkit.to_target(\"file\", \"_from_xr_2.grib\")\n", + "ekd.from_source(\"file\", \"_from_xr_2.grib\").to_fieldlist().head()" ] }, { @@ -1198,7 +1229,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/xarray_engine_remapping.ipynb b/docs/examples/xr_engine/xarray_engine_remapping.ipynb similarity index 65% rename from docs/examples/xarray_engine_remapping.ipynb rename to docs/examples/xr_engine/xarray_engine_remapping.ipynb index 171332fb5..b5adbc176 100644 --- a/docs/examples/xarray_engine_remapping.ipynb +++ b/docs/examples/xr_engine/xarray_engine_remapping.ipynb @@ -66,13 +66,6 @@ "id": "fc6598cb-5516-424d-ba01-a8ee22a120cc", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " " - ] - }, { "data": { "text/html": [ @@ -94,94 +87,80 @@ " \n", " \n", " \n", - " centre\n", - " shortName\n", - " typeOfLevel\n", - " level\n", - " dataDate\n", - " dataTime\n", - " stepRange\n", - " dataType\n", - " number\n", - " gridType\n", + " parameter.variable\n", + " time.valid_datetime\n", + " time.base_datetime\n", + " time.step\n", + " vertical.level\n", + " vertical.level_type\n", + " ensemble.member\n", + " geography.grid_type\n", " \n", " \n", " \n", " \n", " 0\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " cf\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 1\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " cf\n", + " pressure\n", " 0\n", " regular_ll\n", " \n", " \n", " 2\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " pf\n", + " pressure\n", " 1\n", " regular_ll\n", " \n", " \n", " 3\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 00:00:00\n", + " 2024-06-03\n", + " 0 days 00:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 0\n", - " pf\n", + " pressure\n", " 2\n", " regular_ll\n", " \n", " \n", " 4\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " pf\n", + " pressure\n", " 1\n", " regular_ll\n", " \n", " \n", " 5\n", - " ecmf\n", " t\n", - " isobaricInhPa\n", + " 2024-06-03 06:00:00\n", + " 2024-06-03\n", + " 0 days 06:00:00\n", " 500\n", - " 20240603\n", - " 0\n", - " 6\n", - " pf\n", + " pressure\n", " 2\n", " regular_ll\n", " \n", @@ -190,21 +169,21 @@ "" ], "text/plain": [ - " centre shortName typeOfLevel level dataDate dataTime stepRange \\\n", - "0 ecmf t isobaricInhPa 500 20240603 0 0 \n", - "1 ecmf t isobaricInhPa 500 20240603 0 6 \n", - "2 ecmf t isobaricInhPa 500 20240603 0 0 \n", - "3 ecmf t isobaricInhPa 500 20240603 0 0 \n", - "4 ecmf t isobaricInhPa 500 20240603 0 6 \n", - "5 ecmf t isobaricInhPa 500 20240603 0 6 \n", - "\n", - " dataType number gridType \n", - "0 cf 0 regular_ll \n", - "1 cf 0 regular_ll \n", - "2 pf 1 regular_ll \n", - "3 pf 2 regular_ll \n", - "4 pf 1 regular_ll \n", - "5 pf 2 regular_ll " + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2024-06-03 00:00:00 2024-06-03 0 days 00:00:00 \n", + "1 t 2024-06-03 06:00:00 2024-06-03 0 days 06:00:00 \n", + "2 t 2024-06-03 00:00:00 2024-06-03 0 days 00:00:00 \n", + "3 t 2024-06-03 00:00:00 2024-06-03 0 days 00:00:00 \n", + "4 t 2024-06-03 06:00:00 2024-06-03 0 days 06:00:00 \n", + "5 t 2024-06-03 06:00:00 2024-06-03 0 days 06:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 500 pressure 0 regular_ll \n", + "2 500 pressure 1 regular_ll \n", + "3 500 pressure 2 regular_ll \n", + "4 500 pressure 1 regular_ll \n", + "5 500 pressure 2 regular_ll " ] }, "execution_count": 2, @@ -213,7 +192,7 @@ } ], "source": [ - "ds_fl = ekd.from_source(\"sample\", \"ens_cf_pf.grib\")\n", + "ds_fl = ekd.from_source(\"sample\", \"ens_cf_pf.grib\").to_fieldlist()\n", "ds_fl.ls()" ] }, @@ -255,9 +234,7 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 395kB\n",
    -       "Dimensions:                  (number: 3, forecast_reference_time: 4, step: 6,\n",
    +       "Dimensions:                  (member: 3, forecast_reference_time: 4, step: 6,\n",
            "                              latitude: 19, longitude: 36)\n",
            "Coordinates:\n",
    -       "  * number                   (number) int64 24B 0 1 2\n",
    +       "  * member                   (member) <U1 12B '0' '1' '2'\n",
            "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 199...\n",
            "  * step                     (step) int64 48B 1 2 3 4 5 6\n",
            "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
            "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
            "Data variables:\n",
    -       "    2t                       (number, forecast_reference_time, step, latitude, longitude) float64 394kB ...\n",
    -       "Attributes: (12/15)\n",
    -       "    param:        2t\n",
    -       "    paramId:      167\n",
    -       "    class:        c3\n",
    -       "    stream:       msmm\n",
    -       "    levtype:      sfc\n",
    -       "    type:         fcmean\n",
    -       "    ...           ...\n",
    -       "    fcmonth:      1\n",
    -       "    origin:       lfpw\n",
    -       "    domain:       g\n",
    -       "    method:       1\n",
    +       "    2t                       (member, forecast_reference_time, step, latitude, longitude) float64 394kB ...\n",
    +       "Attributes:\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • 2t
      (member, forecast_reference_time, step, latitude, longitude)
      float64
      ...
      standard_name :
      unknown
      long_name :
      2 metre temperature
      units :
      kelvin
      level_type :
      surface
      _earthkit :
      {'message': b"GRIB\\x00\\x00\\x88\\x01\\x00\\x00P\\x80U\\x80\\xff\\x80\\xa7\\x01\\x00\\x00]\\n\\x01\\x00\\x00\\x01\\x02\\xe8\\n\\x00\\x00\\x00\\x14b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x10\\x1fP\\x04\\xc50001\\x00\\x00\\x00\\x08\\x00\\x01\\x00\\x03\\n\\x8e\\x06\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 24, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [49248 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 395kB\n", - "Dimensions: (number: 3, forecast_reference_time: 4, step: 6,\n", + "Dimensions: (member: 3, forecast_reference_time: 4, step: 6,\n", " latitude: 19, longitude: 36)\n", "Coordinates:\n", - " * number (number) int64 24B 0 1 2\n", + " * member (member) \n", "\n", "\n", - "
    <xarray.Dataset> Size: 395kB\n",
    -       "Dimensions:                  (number: 3, forecast_reference_time: 4,\n",
    +       "Dimensions:                  (member: 3, forecast_reference_time: 4,\n",
            "                              forecastMonth: 6, latitude: 19, longitude: 36)\n",
            "Coordinates:\n",
    -       "  * number                   (number) int64 24B 0 1 2\n",
    +       "  * member                   (member) <U1 12B '0' '1' '2'\n",
            "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 199...\n",
            "  * forecastMonth            (forecastMonth) int64 48B 1 2 3 4 5 6\n",
            "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
            "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
            "Data variables:\n",
    -       "    2t                       (number, forecast_reference_time, forecastMonth, latitude, longitude) float64 394kB ...\n",
    -       "Attributes: (12/15)\n",
    -       "    param:        2t\n",
    -       "    paramId:      167\n",
    -       "    class:        c3\n",
    -       "    stream:       msmm\n",
    -       "    levtype:      sfc\n",
    -       "    type:         fcmean\n",
    -       "    ...           ...\n",
    -       "    fcmonth:      1\n",
    -       "    origin:       lfpw\n",
    -       "    domain:       g\n",
    -       "    method:       1\n",
    +       "    2t                       (member, forecast_reference_time, forecastMonth, latitude, longitude) float64 394kB ...\n",
    +       "Attributes:\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • 2t
      (member, forecast_reference_time, forecastMonth, latitude, longitude)
      float64
      ...
      standard_name :
      unknown
      long_name :
      2 metre temperature
      units :
      kelvin
      level_type :
      surface
      _earthkit :
      {'message': b"GRIB\\x00\\x00\\x88\\x01\\x00\\x00P\\x80U\\x80\\xff\\x80\\xa7\\x01\\x00\\x00]\\n\\x01\\x00\\x00\\x01\\x02\\xe8\\n\\x00\\x00\\x00\\x14b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x10\\x1fP\\x04\\xc50001\\x00\\x00\\x00\\x08\\x00\\x01\\x00\\x03\\n\\x8e\\x06\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 24, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [49248 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 395kB\n", - "Dimensions: (number: 3, forecast_reference_time: 4,\n", + "Dimensions: (member: 3, forecast_reference_time: 4,\n", " forecastMonth: 6, latitude: 19, longitude: 36)\n", "Coordinates:\n", - " * number (number) int64 24B 0 1 2\n", + " * member (member) \n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
            "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
            "                              latitude: 19, longitude: 36)\n",
    @@ -527,20 +621,12 @@
            "    u                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • t
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      typeOfLevel :
      hybrid
      _earthkit :
      {'message': b'GRIB\\xff\\xff\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x14\\x00\\x00\\x00\\x15\\x01\\x00b\\x00\\x00\\x1f\\x00\\x01\\x07\\xe8\\x06\\x03\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x11\\x02\\x00\\x01\\x00\\x01\\x00\\t\\x04\\x010001\\x00\\x00\\x00H\\x03\\x00\\x00\\x00\\x02\\xac\\x00\\x00\\x00\\x00\\x06\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\x00\\x00$\\x00\\x00\\x00\\x13\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\x05]J\\x80\\x00\\x00\\x00\\x000\\x85]J\\x80\\x14\\xdc\\x93\\x80\\x00\\x98\\x96\\x80\\x00\\x98\\x96\\x80\\x00\\x00\\x00\\x04r\\x04\\x01\\x14\\x00\\x00\\x00\\x00\\x02\\xff\\x9a\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x00i\\x00\\x00\\x00\\x00Z\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\x00\\x00\\x00@\\x00\\x05\\xfb@F\\x8b\\x1e@\\x95P\\x8f@\\xda~\\xcaA\\x1b\\xf3\\x93AY\\xaf\\xd1A\\x94\\xdf\\x17A\\xc7\\xe2\\xc0B\\x03\\xf1^B+\\x84XB[\\xd2eB\\x8b\\n\\x89B\\xad\\xca\\xb1B\\xd6\\xd4\\xdcC\\x03l\\xeeC\\x1fG\\x87C?V\\xacCc\\xf8\\rC\\x86\\xc5\\x11C\\x9e5\\xdbC\\xb8}\\xbeC\\xd5\\xcb\\xd7C\\xf6N\\xdaD\\r\\x1avD \\xd5\\xc1D6o\\xa0DM\\xfd\\xf1Dg\\x96\\x13D\\x81\\xa6pD\\x90\\x9bQD\\xa0\\xb3\\x88D\\xb1\\xf8\\xa5D\\xc4s\\xefD\\xd8.^D\\xed0\\x9eE\\x01\\xc1\\x89E\\r\\x96\\xe8E\\x1a\\x1cTE\\'U\\x92E5FCEC\\xf1\\xe9ES[\\xe6Ec\\x87~Etw\\xd9E\\x83\\x17rE\\x8cV\\x8aE\\x95\\xf92E\\x9f\\xff)E\\xaag\\xedE\\xb52\\xc2E\\xc0`\\x98E\\xcb\\xf7\\x93E\\xd7\\xfe\\xf7E\\xe4~\\xf4E\\xf1{LE\\xfe\\xfa\\xd5F\\x06\\x82\\x1aF\\r\\xd1\\x9aF\\x15j\\xbbF\\x1dG\\xeaF%b\\x87F-\\xb2\\xa6F60EF>\\xce1FG{~FP2\\xadFX\\xe5SFa\\x80\\x8fFi\\xfevFrQ\\x07FzhvF\\x81\\x1e\\xa5F\\x84\\xe1\\x94F\\x88w:F\\x8b\\xdb>F\\x8f\\x08\\xdeF\\x91\\xfbpF\\x94\\xae\\x94F\\x97\\x1f\\x06F\\x99H\\x16F\\x9b&\\xc8F\\x9c\\xb7\\xddF\\x9d\\xf7TF\\x9e\\xe3\\xbaF\\x9fx\\x9eF\\x9f\\xb4(F\\x9f\\x93pF\\x9f\\x13\\xa2F\\x9e3\\x06F\\x9c\\xee,F\\x9bD\\rF\\x991%F\\x96\\xb4tF\\x93\\xca\\xecF\\x90sjF\\x8c\\xad\\xdaF\\x88\\x7f\\xaeF\\x83\\xf1`F~\\x180Fs\\xb2\\xc8Fh\\xc9\\xd0F]uLFQ\\xcf\\x14FE\\xf1\\x08F9\\xf5\\\\F-\\xf58F"\\x08\\xb4F\\x16F\\x10F\\n\\xc1\\xd0E\\xff\\x1b\\x00E\\xe9r\\xc0E\\xd4\\xa3`E\\xc0\\xc4@E\\xad\\xe3\\x10E\\x9c\\x0e`E\\x8bK\\x00Ew?`EZ\\x13\\xc0E?\\x14@E&2@E\\x0fc\\xe0D\\xf50\\x00D\\xcfo@D\\xadq\\x80D\\x8e\\xe8\\x00Dg\\xa0\\x80D7\\xbf\\x80D\\x0e\\x04\\x00C\\xd45\\x00C\\x97=\\x00CJ|\\x00B\\xf44\\x00B{ \\x00A\\xb6\\xb0\\x00@p\\x80\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x003$\\x11X6\\xe2\\xd9\\xf97\\xcc>\\xef8w#\\x048\\xea\\xb3\\x889P979\\xb2t\\xfd:\\x135W:i:\\x9e:\\xb1P\\x9f;\\x02\\x89\\x7f;;>\\x97;\\x82\\x1e\\xc6;\\xb08a;\\xe9\\xbf\\x1a<\\x17\\xbda<An\\x0c<r\\xbd@<\\x96\\x10\\x94<\\xb7!\\x82<\\xdc\\xe2\\x9a=\\x03\\xcb\\x15=\\x1b\\xc1\\x92=6w\\xec=T\\x0f\\xf1=t\\xa5\\xc8=\\x8c.\\x99=\\x9f\\xa8\\x9b=\\xb4\\xcf\\'=\\xcb\\xb2\\x8f=\\xe4\\\\\\x8d=\\xfe\\xde\\xa9>\\r\\xa1\\xe5>\\x1c\\xcc\\xcf>,\\xf6\\xda>>%`>P`\\x00>c\\xab:>x\\x0e\\xd0>\\x86\\xc7\\xa4>\\x92\\x19\\xec>\\x9e\\x00\\x9e>\\xaav\\xfe>\\xb7m\\x18>\\xc4\\xcbE>\\xd2~\\xf2>\\xe0t\\xd0>\\xee\\x97\\xfc>\\xfc\\xd3e?\\x05\\x88\\xd6?\\x0c\\x9f\\x00?\\x13\\xa2\\x19?\\x1a\\x88\\xae?!J\\x07?\\'\\xdeP?.>\\x9f?4e0?:M\\x16??\\xf2\\xab?ER\\xfd?Jlj?O=\\xc1?S\\xc7\\x0e?X\\x08\\x98?\\\\\\x03\\xb9?_\\xb9\\xd4?c-$?f_\\xe1?iT\\xab?l\\x0ev?n\\x8f\\xbd?p\\xdc,?r\\xf5\\xe3?t\\xe0n?v\\x9e8?x2\\x08?y\\x9e\\xbd?z\\xe5\\xf9?|\\n\\xf0?}\\x0eN?}\\xf2\\xaa?~\\xb8|?\\x7fd\\xb0?\\x80\\x00\\x00\\x00\\x00\\x00\\x15\\x05\\x00\\x00\\x02\\xac\\x00\\x00C\\x88\\x80\\x00\\x80\\n\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x06\\xff\\x00\\x00\\x00\\x05\\x077777', 'bitsPerValue': 24, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      metadata.typeOfLevel :
      hybrid
      [10944 values with dtype=float64]
    • u
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      eastward_wind
      long_name :
      U component of wind
      units :
      meter / second
      typeOfLevel :
      hybrid
      _earthkit :
      {'message': b'GRIB\\xff\\xff\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x14\\x00\\x00\\x00\\x15\\x01\\x00b\\x00\\x00\\x1f\\x00\\x01\\x07\\xe8\\x06\\x03\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x11\\x02\\x00\\x01\\x00\\x01\\x00\\t\\x04\\x010001\\x00\\x00\\x00H\\x03\\x00\\x00\\x00\\x02\\xac\\x00\\x00\\x00\\x00\\x06\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\x00\\x00$\\x00\\x00\\x00\\x13\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\x05]J\\x80\\x00\\x00\\x00\\x000\\x85]J\\x80\\x14\\xdc\\x93\\x80\\x00\\x98\\x96\\x80\\x00\\x98\\x96\\x80\\x00\\x00\\x00\\x04r\\x04\\x01\\x14\\x00\\x00\\x02\\x02\\x02\\xff\\x9a\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x00i\\x00\\x00\\x00\\x00Z\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\x00\\x00\\x00@\\x00\\x05\\xfb@F\\x8b\\x1e@\\x95P\\x8f@\\xda~\\xcaA\\x1b\\xf3\\x93AY\\xaf\\xd1A\\x94\\xdf\\x17A\\xc7\\xe2\\xc0B\\x03\\xf1^B+\\x84XB[\\xd2eB\\x8b\\n\\x89B\\xad\\xca\\xb1B\\xd6\\xd4\\xdcC\\x03l\\xeeC\\x1fG\\x87C?V\\xacCc\\xf8\\rC\\x86\\xc5\\x11C\\x9e5\\xdbC\\xb8}\\xbeC\\xd5\\xcb\\xd7C\\xf6N\\xdaD\\r\\x1avD \\xd5\\xc1D6o\\xa0DM\\xfd\\xf1Dg\\x96\\x13D\\x81\\xa6pD\\x90\\x9bQD\\xa0\\xb3\\x88D\\xb1\\xf8\\xa5D\\xc4s\\xefD\\xd8.^D\\xed0\\x9eE\\x01\\xc1\\x89E\\r\\x96\\xe8E\\x1a\\x1cTE\\'U\\x92E5FCEC\\xf1\\xe9ES[\\xe6Ec\\x87~Etw\\xd9E\\x83\\x17rE\\x8cV\\x8aE\\x95\\xf92E\\x9f\\xff)E\\xaag\\xedE\\xb52\\xc2E\\xc0`\\x98E\\xcb\\xf7\\x93E\\xd7\\xfe\\xf7E\\xe4~\\xf4E\\xf1{LE\\xfe\\xfa\\xd5F\\x06\\x82\\x1aF\\r\\xd1\\x9aF\\x15j\\xbbF\\x1dG\\xeaF%b\\x87F-\\xb2\\xa6F60EF>\\xce1FG{~FP2\\xadFX\\xe5SFa\\x80\\x8fFi\\xfevFrQ\\x07FzhvF\\x81\\x1e\\xa5F\\x84\\xe1\\x94F\\x88w:F\\x8b\\xdb>F\\x8f\\x08\\xdeF\\x91\\xfbpF\\x94\\xae\\x94F\\x97\\x1f\\x06F\\x99H\\x16F\\x9b&\\xc8F\\x9c\\xb7\\xddF\\x9d\\xf7TF\\x9e\\xe3\\xbaF\\x9fx\\x9eF\\x9f\\xb4(F\\x9f\\x93pF\\x9f\\x13\\xa2F\\x9e3\\x06F\\x9c\\xee,F\\x9bD\\rF\\x991%F\\x96\\xb4tF\\x93\\xca\\xecF\\x90sjF\\x8c\\xad\\xdaF\\x88\\x7f\\xaeF\\x83\\xf1`F~\\x180Fs\\xb2\\xc8Fh\\xc9\\xd0F]uLFQ\\xcf\\x14FE\\xf1\\x08F9\\xf5\\\\F-\\xf58F"\\x08\\xb4F\\x16F\\x10F\\n\\xc1\\xd0E\\xff\\x1b\\x00E\\xe9r\\xc0E\\xd4\\xa3`E\\xc0\\xc4@E\\xad\\xe3\\x10E\\x9c\\x0e`E\\x8bK\\x00Ew?`EZ\\x13\\xc0E?\\x14@E&2@E\\x0fc\\xe0D\\xf50\\x00D\\xcfo@D\\xadq\\x80D\\x8e\\xe8\\x00Dg\\xa0\\x80D7\\xbf\\x80D\\x0e\\x04\\x00C\\xd45\\x00C\\x97=\\x00CJ|\\x00B\\xf44\\x00B{ \\x00A\\xb6\\xb0\\x00@p\\x80\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x003$\\x11X6\\xe2\\xd9\\xf97\\xcc>\\xef8w#\\x048\\xea\\xb3\\x889P979\\xb2t\\xfd:\\x135W:i:\\x9e:\\xb1P\\x9f;\\x02\\x89\\x7f;;>\\x97;\\x82\\x1e\\xc6;\\xb08a;\\xe9\\xbf\\x1a<\\x17\\xbda<An\\x0c<r\\xbd@<\\x96\\x10\\x94<\\xb7!\\x82<\\xdc\\xe2\\x9a=\\x03\\xcb\\x15=\\x1b\\xc1\\x92=6w\\xec=T\\x0f\\xf1=t\\xa5\\xc8=\\x8c.\\x99=\\x9f\\xa8\\x9b=\\xb4\\xcf\\'=\\xcb\\xb2\\x8f=\\xe4\\\\\\x8d=\\xfe\\xde\\xa9>\\r\\xa1\\xe5>\\x1c\\xcc\\xcf>,\\xf6\\xda>>%`>P`\\x00>c\\xab:>x\\x0e\\xd0>\\x86\\xc7\\xa4>\\x92\\x19\\xec>\\x9e\\x00\\x9e>\\xaav\\xfe>\\xb7m\\x18>\\xc4\\xcbE>\\xd2~\\xf2>\\xe0t\\xd0>\\xee\\x97\\xfc>\\xfc\\xd3e?\\x05\\x88\\xd6?\\x0c\\x9f\\x00?\\x13\\xa2\\x19?\\x1a\\x88\\xae?!J\\x07?\\'\\xdeP?.>\\x9f?4e0?:M\\x16??\\xf2\\xab?ER\\xfd?Jlj?O=\\xc1?S\\xc7\\x0e?X\\x08\\x98?\\\\\\x03\\xb9?_\\xb9\\xd4?c-$?f_\\xe1?iT\\xab?l\\x0ev?n\\x8f\\xbd?p\\xdc,?r\\xf5\\xe3?t\\xe0n?v\\x9e8?x2\\x08?y\\x9e\\xbd?z\\xe5\\xf9?|\\n\\xf0?}\\x0eN?}\\xf2\\xaa?~\\xb8|?\\x7fd\\xb0?\\x80\\x00\\x00\\x00\\x00\\x00\\x15\\x05\\x00\\x00\\x02\\xac\\x00\\x00C\\x88\\x80\\x00\\x80\\n\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x06\\xff\\x00\\x00\\x00\\x05\\x077777', 'bitsPerValue': 24, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      metadata.typeOfLevel :
      hybrid
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", @@ -592,7 +678,7 @@ { "data": { "text/plain": [ - "{'typeOfLevel': 'hybrid'}" + "{'metadata.typeOfLevel': 'hybrid'}" ] }, "execution_count": 4, @@ -635,7 +721,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/xarray_engine_squeeze.ipynb b/docs/examples/xr_engine/xarray_engine_squeeze.ipynb similarity index 59% rename from docs/examples/xarray_engine_squeeze.ipynb rename to docs/examples/xr_engine/xarray_engine_squeeze.ipynb index 02dcca80a..82288c6ba 100644 --- a/docs/examples/xarray_engine_squeeze.ipynb +++ b/docs/examples/xr_engine/xarray_engine_squeeze.ipynb @@ -39,18 +39,10 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " " - ] - } - ], + "outputs": [], "source": [ "import earthkit.data as ekd\n", - "ds_fl = ekd.from_source(\"sample\", \"pl.grib\")" + "ds_fl = ekd.from_source(\"sample\", \"pl.grib\").to_fieldlist()" ] }, { @@ -65,7 +57,7 @@ "tags": [] }, "source": [ - "By default, ``squeeze=True`` in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray`. This means that if a dimension has only one value, it is removed from the dataset. E.g. in the following example the dimensions ``\"number\"`` and ``\"level_type\"`` are removed:" + "By default, ``squeeze=True`` in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray`. This means that if a dimension has only one value, it is removed from the dataset. E.g. in the following example the dimensions ``\"member\"`` and ``\"level_type\"`` are removed:" ] }, { @@ -98,9 +90,7 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
    -       "Dimensions:                  (number: 1, forecast_reference_time: 4, step: 2,\n",
    +       "Dimensions:                  (member: 1, forecast_reference_time: 4, step: 2,\n",
            "                              level: 2, level_type: 1, latitude: 19,\n",
            "                              longitude: 36)\n",
            "Coordinates:\n",
    -       "  * number                   (number) int64 8B 0\n",
    +       "  * member                   (member) <U1 4B '0'\n",
            "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
            "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
            "  * level                    (level) int64 16B 500 700\n",
    -       "  * level_type               (level_type) <U2 8B 'pl'\n",
    +       "  * level_type               (level_type) <U8 32B 'pressure'\n",
            "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
            "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
            "Data variables:\n",
    -       "    r                        (number, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    -       "    t                        (number, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    +       "    r                        (member, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    +       "    t                        (member, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    date:         20240603\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
    -       "    levelist:     500\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • r
      (member, forecast_reference_time, step, level, level_type, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (member, forecast_reference_time, step, level, level_type, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", - "Dimensions: (number: 1, forecast_reference_time: 4, step: 2,\n", + "Dimensions: (member: 1, forecast_reference_time: 4, step: 2,\n", " level: 2, level_type: 1, latitude: 19,\n", " longitude: 36)\n", "Coordinates:\n", - " * number (number) int64 8B 0\n", + " * member (member) \n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
    -       "Dimensions:                  (number: 1, forecast_reference_time: 4, step: 2,\n",
    +       "Dimensions:                  (member: 1, forecast_reference_time: 4, step: 2,\n",
            "                              level: 2, level_type: 1, latitude: 19,\n",
            "                              longitude: 36)\n",
            "Coordinates:\n",
    -       "  * number                   (number) int64 8B 0\n",
    +       "  * member                   (member) <U1 4B '0'\n",
            "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
            "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
            "  * level                    (level) int64 16B 500 700\n",
    -       "  * level_type               (level_type) <U2 8B 'pl'\n",
    +       "  * level_type               (level_type) <U8 32B 'pressure'\n",
            "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
            "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
            "Data variables:\n",
    -       "    r                        (number, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    -       "    t                        (number, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    +       "    r                        (member, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
    +       "    t                        (member, forecast_reference_time, step, level, level_type, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    date:         20240603\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
    -       "    levelist:     500\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • r
      (member, forecast_reference_time, step, level, level_type, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (member, forecast_reference_time, step, level, level_type, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", - "Dimensions: (number: 1, forecast_reference_time: 4, step: 2,\n", + "Dimensions: (member: 1, forecast_reference_time: 4, step: 2,\n", " level: 2, level_type: 1, latitude: 19,\n", " longitude: 36)\n", "Coordinates:\n", - " * number (number) int64 8B 0\n", + " * member (member) \n", " \n", " \n", - " param\n", - " step\n", - " stepRange\n", - " startStep\n", - " endStep\n", + " metadata.param\n", + " metadata.step\n", + " metadata.stepRange\n", + " metadata.startStep\n", + " metadata.endStep\n", " \n", " \n", " \n", @@ -98,9 +98,13 @@ "" ], "text/plain": [ - " param step stepRange startStep endStep\n", - "0 lsp 71-72 71-72 71 72\n", - "1 lsp 72-73 72-73 72 73" + " metadata.param metadata.step metadata.stepRange metadata.startStep \\\n", + "0 lsp 71-72 71-72 71 \n", + "1 lsp 72-73 72-73 72 \n", + "\n", + " metadata.endStep \n", + "0 72 \n", + "1 73 " ] }, "execution_count": 1, @@ -110,8 +114,8 @@ ], "source": [ "import earthkit.data as ekd\n", - "ds_fl = ekd.from_source(\"sample\", \"lsp_step_range.grib2\")\n", - "ds_fl.ls(keys=[\"param\", \"step\", \"stepRange\", \"startStep\", \"endStep\"])" + "ds_fl = ekd.from_source(\"sample\", \"lsp_step_range.grib2\").to_fieldlist()\n", + "ds_fl.ls(keys=[\"metadata.param\", \"metadata.step\", \"metadata.stepRange\", \"metadata.startStep\", \"metadata.endStep\"])" ] }, { @@ -153,39 +157,86 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 2kB\n",
            "Dimensions:    (step: 2, latitude: 7, longitude: 12)\n",
            "Coordinates:\n",
    @@ -517,20 +629,8 @@
            "Data variables:\n",
            "    lsp        (step, latitude, longitude) float64 1kB ...\n",
            "Attributes:\n",
    -       "    param:        lsp\n",
    -       "    paramId:      142\n",
    -       "    class:        d1\n",
    -       "    stream:       oper\n",
    -       "    levtype:      sfc\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    date:         20250527\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    " + " institution: ECMWF" ], "text/plain": [ " Size: 2kB\n", @@ -542,16 +642,6 @@ "Data variables:\n", " lsp (step, latitude, longitude) float64 1kB ...\n", "Attributes:\n", - " param: lsp\n", - " paramId: 142\n", - " class: d1\n", - " stream: oper\n", - " levtype: sfc\n", - " type: fc\n", - " expver: 0001\n", - " date: 20250527\n", - " time: 0\n", - " domain: g\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -634,7 +724,7 @@ } ], "source": [ - "ds = ds_fl.to_xarray(dim_roles={\"step\": \"startStep\"})\n", + "ds = ds_fl.to_xarray(dim_roles={\"step\": \"metadata.startStep\"})\n", "[int(x* 1E-9/(3600)) for x in ds[\"step\"].values]" ] }, @@ -663,7 +753,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/xarray_engine_temporal.ipynb b/docs/examples/xr_engine/xarray_engine_temporal.ipynb similarity index 55% rename from docs/examples/xarray_engine_temporal.ipynb rename to docs/examples/xr_engine/xarray_engine_temporal.ipynb index 7081b9611..5c0b8929c 100644 --- a/docs/examples/xarray_engine_temporal.ipynb +++ b/docs/examples/xr_engine/xarray_engine_temporal.ipynb @@ -54,25 +54,10 @@ }, "tags": [] }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8dca45f059a04a48898e26443d3b1a64", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "pl.grib: 0%| | 0.00/48.8k [00:00\n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
            "Dimensions:    (date: 2, time: 2, step: 2, level: 2, latitude: 19, longitude: 36)\n",
            "Coordinates:\n",
    @@ -539,25 +632,12 @@
            "    r          (date, time, step, level, latitude, longitude) float64 88kB ...\n",
            "    t          (date, time, step, level, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • r
      (date, time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (date, time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", @@ -573,13 +653,6 @@ " r (date, time, step, level, latitude, longitude) float64 88kB ...\n", " t (date, time, step, level, latitude, longitude) float64 88kB ...\n", "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " domain: g\n", - " number: 0\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -653,39 +726,86 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
            "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
            "                              latitude: 19, longitude: 36)\n",
    @@ -1021,30 +1202,13 @@
            "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
            "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    date:         20240603\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • r
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (forecast_reference_time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", @@ -1060,15 +1224,6 @@ " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -1127,7 +1282,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "98fb8660778641739b5bdba816ad80ef", + "model_id": "a2c28291979845549eab205d510d81a0", "version_major": 2, "version_minor": 0 }, @@ -1156,39 +1311,86 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 6kB\n",
            "Dimensions:     (valid_time: 8, latitude: 7, longitude: 12)\n",
            "Coordinates:\n",
    @@ -1519,30 +1782,13 @@
            "  * longitude   (longitude) float64 96B -70.0 -60.0 -50.0 ... 20.0 30.0 40.0\n",
            "Data variables:\n",
            "    msl         (valid_time, latitude, longitude) float64 5kB ...\n",
    -       "Attributes: (12/13)\n",
    -       "    param:        msl\n",
    -       "    paramId:      151\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      sfc\n",
    -       "    type:         an\n",
    -       "    ...           ...\n",
    -       "    date:         20160925\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
    +       "Attributes:\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • msl
      (valid_time, latitude, longitude)
      float64
      ...
      standard_name :
      air_pressure_at_mean_sea_level
      long_name :
      Mean sea level pressure
      units :
      pascal
      level_type :
      surface
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x92\\xff\\x80\\x97\\x01\\x00\\x00\\x10\\t\\x19\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\x02\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00\\x0c\\x00\\x07\\x018\\x80\\x81\\x11p\\x80\\x00N \\x00\\x9c@'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 12, 'grid_spec': '{"grid": [10.0, 10.0], "area": [80.0, -70.0, 20.0, 40.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [80.0, -70.0, 20.0, 40.0]}
      [672 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 6kB\n", @@ -1553,18 +1799,7 @@ " * longitude (longitude) float64 96B -70.0 -60.0 -50.0 ... 20.0 30.0 40.0\n", "Data variables:\n", " msl (valid_time, latitude, longitude) float64 5kB ...\n", - "Attributes: (12/13)\n", - " param: msl\n", - " paramId: 151\n", - " class: od\n", - " stream: oper\n", - " levtype: sfc\n", - " type: an\n", - " ... ...\n", - " date: 20160925\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", + "Attributes:\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -1624,39 +1859,86 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 44kB\n",
    -       "Dimensions:     (valid_time: 2, level: 2, latitude: 19, longitude: 36)\n",
    -       "Coordinates:\n",
    -       "  * valid_time  (valid_time) datetime64[ns] 16B 2024-06-03 2024-06-03T06:00:00\n",
    -       "  * level       (level) int64 16B 500 700\n",
    -       "  * latitude    (latitude) float64 152B 90.0 80.0 70.0 ... -70.0 -80.0 -90.0\n",
    -       "  * longitude   (longitude) float64 288B 0.0 10.0 20.0 ... 330.0 340.0 350.0\n",
    +       "\n",
    +       ".xr-var-attrs-in:checked + label > .xr-icon-file-text2,\n",
    +       ".xr-var-data-in:checked + label > .xr-icon-database,\n",
    +       ".xr-index-data-in:checked + label > .xr-icon-database {\n",
    +       "  color: var(--xr-font-color0);\n",
    +       "  filter: drop-shadow(1px 1px 5px var(--xr-font-color2));\n",
    +       "  stroke-width: 0.8px;\n",
    +       "}\n",
    +       "
    <xarray.Dataset> Size: 0B\n",
    +       "Dimensions:  ()\n",
            "Data variables:\n",
    -       "    r           (valid_time, level, latitude, longitude) float64 22kB ...\n",
    -       "    t           (valid_time, level, latitude, longitude) float64 22kB ...\n",
    -       "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    date:         20240603\n",
    -       "    time:         0\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
    -       "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    " + " *empty*
    " ], "text/plain": [ - " Size: 44kB\n", - "Dimensions: (valid_time: 2, level: 2, latitude: 19, longitude: 36)\n", - "Coordinates:\n", - " * valid_time (valid_time) datetime64[ns] 16B 2024-06-03 2024-06-03T06:00:00\n", - " * level (level) int64 16B 500 700\n", - " * latitude (latitude) float64 152B 90.0 80.0 70.0 ... -70.0 -80.0 -90.0\n", - " * longitude (longitude) float64 288B 0.0 10.0 20.0 ... 330.0 340.0 350.0\n", + " Size: 0B\n", + "Dimensions: ()\n", "Data variables:\n", - " r (valid_time, level, latitude, longitude) float64 22kB ...\n", - " t (valid_time, level, latitude, longitude) float64 22kB ...\n", - "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " date: 20240603\n", - " time: 0\n", - " domain: g\n", - " number: 0\n", - " Conventions: CF-1.8\n", - " institution: ECMWF" + " *empty*" ] }, "execution_count": 5, @@ -2105,39 +2402,86 @@ "\n", "\n", "\n", - "
    <xarray.Dataset> Size: 176kB\n",
            "Dimensions:     (date: 2, time: 2, step: 2, level: 2, latitude: 19,\n",
            "                 longitude: 36)\n",
    @@ -2468,32 +2873,19 @@
            "  * time        (time) timedelta64[ns] 16B 00:00:00 12:00:00\n",
            "  * step        (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
            "  * level       (level) int64 16B 500 700\n",
    -       "    valid_time  (date, time, step) datetime64[ns] 64B ...\n",
            "  * latitude    (latitude) float64 152B 90.0 80.0 70.0 ... -70.0 -80.0 -90.0\n",
            "  * longitude   (longitude) float64 288B 0.0 10.0 20.0 ... 330.0 340.0 350.0\n",
    +       "    valid_time  (date, time, step) datetime64[ns] 64B ...\n",
            "Data variables:\n",
            "    r           (date, time, step, level, latitude, longitude) float64 88kB ...\n",
            "    t           (date, time, step, level, latitude, longitude) float64 88kB ...\n",
            "Attributes:\n",
    -       "    class:        od\n",
    -       "    stream:       oper\n",
    -       "    levtype:      pl\n",
    -       "    type:         fc\n",
    -       "    expver:       0001\n",
    -       "    domain:       g\n",
    -       "    number:       0\n",
            "    Conventions:  CF-1.8\n",
    -       "    institution:  ECMWF
    • r
      (date, time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      relative_humidity
      long_name :
      Relative humidity
      units :
      percent
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x9dd\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
    • t
      (date, time, step, level, latitude, longitude)
      float64
      ...
      standard_name :
      air_temperature
      long_name :
      Temperature
      units :
      kelvin
      level_type :
      pressure
      _earthkit :
      {'message': b"GRIB\\x00\\x00l\\x01\\x00\\x004\\x80b\\x9a\\xff\\x80\\x82d\\x01\\xf4\\x18\\x06\\x03\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\t\\x04\\x010001\\x00\\x00\\x00\\x00\\x00 \\x00\\xff\\x00\\x00$\\x00\\x13\\x01_\\x90\\x00\\x00\\x00\\x80\\x81_\\x90\\x05W0'\\x10'\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x80\\x02D\\xb9}n\\x00\\x007777", 'bitsPerValue': 16, 'grid_spec': '{"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}'}
      ek_grid_spec :
      {"grid": [10.0, 10.0], "area": [90.0, 0.0, -90.0, 350.0]}
      [10944 values with dtype=float64]
  • Conventions :
    CF-1.8
    institution :
    ECMWF
  • " ], "text/plain": [ " Size: 176kB\n", @@ -2504,20 +2896,13 @@ " * time (time) timedelta64[ns] 16B 00:00:00 12:00:00\n", " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", " * level (level) int64 16B 500 700\n", - " valid_time (date, time, step) datetime64[ns] 64B ...\n", " * latitude (latitude) float64 152B 90.0 80.0 70.0 ... -70.0 -80.0 -90.0\n", " * longitude (longitude) float64 288B 0.0 10.0 20.0 ... 330.0 340.0 350.0\n", + " valid_time (date, time, step) datetime64[ns] 64B ...\n", "Data variables:\n", " r (date, time, step, level, latitude, longitude) float64 88kB ...\n", " t (date, time, step, level, latitude, longitude) float64 88kB ...\n", "Attributes:\n", - " class: od\n", - " stream: oper\n", - " levtype: pl\n", - " type: fc\n", - " expver: 0001\n", - " domain: g\n", - " number: 0\n", " Conventions: CF-1.8\n", " institution: ECMWF" ] @@ -2622,9 +3007,9 @@ "data": { "text/plain": [ "Coordinates:\n", - " * date (date) int64 16B 20240603 20240604\n", - " * time (time) int64 16B 0 1200\n", - " * step (step) int64 16B 0 6\n", + " * date (date) object 16B 2024-06-03 2024-06-04\n", + " * time (time) object 16B 00:00:00 12:00:00\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", " * level (level) int64 16B 500 700\n", " * latitude (latitude) float64 152B 90.0 80.0 70.0 60.0 ... -70.0 -80.0 -90.0\n", " * longitude (longitude) float64 288B 0.0 10.0 20.0 30.0 ... 330.0 340.0 350.0" @@ -2655,7 +3040,8 @@ { "data": { "text/plain": [ - "{'units': 'hours'}" + "{'standard_name': 'forecast_period',\n", + " 'long_name': 'time since forecast_reference_time'}" ] }, "execution_count": 9, @@ -2692,7 +3078,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.12" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/docs/examples/xr_engine/xarray_engine_to_grib.ipynb b/docs/examples/xr_engine/xarray_engine_to_grib.ipynb new file mode 100644 index 000000000..9170a44fd --- /dev/null +++ b/docs/examples/xr_engine/xarray_engine_to_grib.ipynb @@ -0,0 +1,1790 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3cd3659f-bd6d-49d9-821a-9183cbe84655", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Xarray engine: writing back to GRIB" + ] + }, + { + "cell_type": "markdown", + "id": "4394dbab-dd68-4523-8581-28fc4001048d", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "
    \n", + "Warning: converting back Xarray to GRIB is an experimental feature and is not yet fully supported.
    " + ] + }, + { + "cell_type": "markdown", + "id": "359a59ce-e285-4202-84ed-995efeea4dda", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "First, we get some example GRIB data and convert it into Xarray." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a2ef916d-79aa-4c59-9bff-b92c0dafca1f", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 176kB\n",
    +       "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
    +       "                              latitude: 19, longitude: 36)\n",
    +       "Coordinates:\n",
    +       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
    +       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
    +       "  * level                    (level) int64 16B 500 700\n",
    +       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
    +       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
    +       "Data variables:\n",
    +       "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
    +       "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
    +       "Attributes:\n",
    +       "    Conventions:  CF-1.8\n",
    +       "    institution:  ECMWF
    " + ], + "text/plain": [ + " Size: 176kB\n", + "Dimensions: (forecast_reference_time: 4, step: 2, level: 2,\n", + " latitude: 19, longitude: 36)\n", + "Coordinates:\n", + " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", + " * level (level) int64 16B 500 700\n", + " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", + " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", + "Data variables:\n", + " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", + " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import earthkit.data as ekd\n", + "\n", + "ds_fl = ekd.from_source(\"sample\", \"pl.grib\").to_fieldlist()\n", + "ds_xr = ds_fl.to_xarray()\n", + "ds_xr" + ] + }, + { + "cell_type": "raw", + "id": "28af4ff2-aaf3-4952-855d-3b91588e2de5", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "By default, ``add_earthkit_attrs=True`` in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` and some special earthkit attributes are added to the dataset. This is needed for the Xarray to GRIB conversion. For this reason, if the Xarray is modified we must ensure the variable attributes are copied to the new Xarray dataset. By default, variable attributes are not kept in Xarray computations so we need to set the global Xarray ``keep_attrs`` option to enable it." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b98402ad-8e81-42af-9426-c8c59d0d1a45", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# ensure earthkit attributes are set\n", + "import xarray as xr \n", + "xr.set_options(keep_attrs=True)\n", + "\n", + "# modify values\n", + "ds_xr += 1" + ] + }, + { + "cell_type": "markdown", + "id": "40c77039-21e0-48f4-a809-9530672da724", + "metadata": {}, + "source": [ + "#### Using to_target()" + ] + }, + { + "cell_type": "raw", + "id": "d6c82521-859e-4bc9-ad3e-686071af88e8", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "It is possible to directly write the Xarray dataset created with the earthkit engine into a GRIB file with :func:`to_target`. This is a memory efficient way to write GRIB to disk since only one field is loaded into memory at a time. We can call :func:`to_target` either on the ``earthkit`` accessor or as a top level function." + ] + }, + { + "cell_type": "markdown", + "id": "fcf81205-624e-4d4c-ad86-0595ff3e588f", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "First, we write a datarray into a GRIB file." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "68d4ec97-2429-44f8-8f80-5ffd58639222", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    Field
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "\n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    number_of_values684
    array_typendarray
    array_dtypefloat64
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    variablet
    unitskelvin
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    valid_datetime2024-06-03 00:00:00
    base_datetime2024-06-03 00:00:00
    step0:00:00
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    level500
    layerNone
    level_typepressure
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    member0
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    grid_spec{'grid': [10.0, 10.0], 'area': [90.0, 0.0, -90.0, 350.0]}
    grid_typeregular_ll
    shape(19, 36)
    area[90.0, 0.0, -90.0, 350.0]
    \n", + "
    \n", + " \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + " " + ], + "text/plain": [ + "Field(t, 2024-06-03 00:00:00, 2024-06-03 00:00:00, 0:00:00, 500, pressure, 0, regular_ll)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# option1: writing to GRIB file using the accessor\n", + "ds_xr[\"t\"].earthkit.to_target(\"file\", \"_from_xr_1.grib\")\n", + "\n", + "# option2: writing to GRIB file using the top level function\n", + "ekd.to_target(\"file\", \"_from_xr_1a.grib\", data=ds_xr[\"t\"])\n", + "\n", + "# check the results\n", + "ds_tmp1 = ekd.from_source(\"file\", \"_from_xr_1.grib\").to_fieldlist()\n", + "ds_tmp1[0]" + ] + }, + { + "cell_type": "markdown", + "id": "d31c0a87-ce0f-4365-a9d4-f9fb88e67eaa", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "Next, we write the whole dataset into a GRIB file." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ced38f9d-20c6-43d7-9e91-32b7bff93a6a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
    0r2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00500pressure0regular_ll
    1r2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00700pressure0regular_ll
    2r2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00500pressure0regular_ll
    3r2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00700pressure0regular_ll
    4r2024-06-03 12:00:002024-06-03 12:00:000 days 00:00:00500pressure0regular_ll
    \n", + "
    " + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 r 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# option1: writing to GRIB file using the accessor\n", + "ds_xr.earthkit.to_target(\"file\", \"_from_xr_2.grib\")\n", + "\n", + "# option2: writing to GRIB file using the top level function\n", + "ekd.to_target(\"file\", \"_from_xr_2a.grib\", data=ds_xr)\n", + "\n", + "# check the results\n", + "ds_tmp2 = ekd.from_source(\"file\", \"_from_xr_2.grib\").to_fieldlist()\n", + "ds_tmp2.head()" + ] + }, + { + "cell_type": "markdown", + "id": "c75cbad6-a383-4bf2-add9-98a6c313aada", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We check if the computation results were correctly written to the generated GRIB data." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0c0bafa4-e3ca-4361-b28c-851b8447f323", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "250.22500610351562\n", + "251.22500610351562\n" + ] + } + ], + "source": [ + "# original GRIB data\n", + "print(ds_fl.sel({\"parameter.variable\": \"t\", \"time.step\": 0, \"vertical.level\": 500})[0].values[0])\n", + "# GRIB data converted from the modified xarray object\n", + "print(ds_tmp1.sel({\"parameter.variable\": \"t\", \"time.step\": 0, \"vertical.level\": 500})[0].values[0])" + ] + }, + { + "cell_type": "markdown", + "id": "be7471ec-bb6c-4873-bd32-babb4b4fc2ba", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Using to_fieldlist()" + ] + }, + { + "cell_type": "raw", + "id": "30ad2d99-db56-4302-99cf-277602202459", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We can also convert the Xarray dataset into a GRIB fieldlist by using :py:meth:`~data.xr_engine.engine.XarrayEarthkit.to_fieldlist` on the ``earthkit`` accessor of the Xarray object. Please note that this will generate a fieldlist entirely stored in memory." + ] + }, + { + "cell_type": "markdown", + "id": "07ee1067-6476-450b-bc12-d90b1dc2fc05", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "First, we convert a dataarray to a GRIB fieldlist." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3784d2d3-8254-48bf-8ee2-9aa2b5d69686", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
    0t2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00500pressure0regular_ll
    1t2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00700pressure0regular_ll
    2t2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00500pressure0regular_ll
    3t2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00700pressure0regular_ll
    4t2024-06-03 12:00:002024-06-03 12:00:000 days 00:00:00500pressure0regular_ll
    \n", + "
    " + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 t 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 t 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 t 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 t 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 t 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fl1 = ds_xr[\"t\"].earthkit.to_fieldlist()\n", + "ds_fl1.head()" + ] + }, + { + "cell_type": "markdown", + "id": "b81cc6de-0b91-442b-9df3-4f3c332aa09c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "Next, we convert back the whole dataset into a GRIB fieldlist." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b39c4d26-dd9f-4a76-9559-6fb8b6676698", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    parameter.variabletime.valid_datetimetime.base_datetimetime.stepvertical.levelvertical.level_typeensemble.membergeography.grid_type
    0r2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00500pressure0regular_ll
    1r2024-06-03 00:00:002024-06-03 00:00:000 days 00:00:00700pressure0regular_ll
    2r2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00500pressure0regular_ll
    3r2024-06-03 06:00:002024-06-03 00:00:000 days 06:00:00700pressure0regular_ll
    4r2024-06-03 12:00:002024-06-03 12:00:000 days 00:00:00500pressure0regular_ll
    \n", + "
    " + ], + "text/plain": [ + " parameter.variable time.valid_datetime time.base_datetime time.step \\\n", + "0 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "1 r 2024-06-03 00:00:00 2024-06-03 00:00:00 0 days 00:00:00 \n", + "2 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "3 r 2024-06-03 06:00:00 2024-06-03 00:00:00 0 days 06:00:00 \n", + "4 r 2024-06-03 12:00:00 2024-06-03 12:00:00 0 days 00:00:00 \n", + "\n", + " vertical.level vertical.level_type ensemble.member geography.grid_type \n", + "0 500 pressure 0 regular_ll \n", + "1 700 pressure 0 regular_ll \n", + "2 500 pressure 0 regular_ll \n", + "3 700 pressure 0 regular_ll \n", + "4 500 pressure 0 regular_ll " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fl2 = ds_xr.earthkit.to_fieldlist()\n", + "ds_fl2.head()" + ] + }, + { + "cell_type": "raw", + "id": "d67d36c7-e0e4-41d5-b0b6-e8b8743f3d10", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "The generated GRIB fieldlist can be saved to disk using the :func:`to_target` method." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "36423326-cf54-494b-8932-72d986b908bf", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    Field
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "\n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    number_of_values684
    array_typendarray
    array_dtypefloat64
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    variablet
    unitskelvin
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    valid_datetime2024-06-03 00:00:00
    base_datetime2024-06-03 00:00:00
    step0:00:00
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    level500
    layerNone
    level_typepressure
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    member0
    \n", + "
    \n", + " \n", + "\n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + "
    grid_spec{'grid': [10.0, 10.0], 'area': [90.0, 0.0, -90.0, 350.0]}
    grid_typeregular_ll
    shape(19, 36)
    area[90.0, 0.0, -90.0, 350.0]
    \n", + "
    \n", + " \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + "
    \n", + " " + ], + "text/plain": [ + "Field(t, 2024-06-03 00:00:00, 2024-06-03 00:00:00, 0:00:00, 500, pressure, 0, regular_ll)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "out_name = \"_from_xr_3.grib\"\n", + "ds_fl1.to_target(\"file\", out_name)\n", + "# read back and check the saved GRIB\n", + "ds_tmp = ekd.from_source(\"file\", out_name).to_fieldlist()\n", + "ds_tmp[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c7a3570-2514-42f9-8b7c-0a087fcc9205", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/xr_engine/xarray_engine_variable_key.ipynb b/docs/examples/xr_engine/xarray_engine_variable_key.ipynb new file mode 100644 index 000000000..0af361bc8 --- /dev/null +++ b/docs/examples/xr_engine/xarray_engine_variable_key.ipynb @@ -0,0 +1,2349 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bc9477f3-1aa0-4539-acef-0bd2b43191ba", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Xarray engine: variable key" + ] + }, + { + "cell_type": "raw", + "id": "236700a8-2212-4675-bd0d-d91159219a6a", + "metadata": { + "editable": true, + "raw_mimetype": "text/restructuredtext", + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "The ``variable_key`` option in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` controls what metadata key should be used to form the DataArray variables. By default it is \"param\", which is an alias to \"shortName\" for GRIB in earthkit-data.\n", + "\n", + "Please note it is also possible to generate an Xarray with a single dataarray containing all the parameters from a GRIB fieldlist. See e.g. the :ref:`/examples/xarray_engine_mono_variable.ipynb` notebook for details." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "08b75c56-0b2f-4cc6-9637-b28ad2aa4455", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 176kB\n",
    +       "Dimensions:                  (forecast_reference_time: 4, step: 2, level: 2,\n",
    +       "                              latitude: 19, longitude: 36)\n",
    +       "Coordinates:\n",
    +       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
    +       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
    +       "  * level                    (level) int64 16B 500 700\n",
    +       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
    +       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
    +       "Data variables:\n",
    +       "    r                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
    +       "    t                        (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n",
    +       "Attributes:\n",
    +       "    Conventions:  CF-1.8\n",
    +       "    institution:  ECMWF
    " + ], + "text/plain": [ + " Size: 176kB\n", + "Dimensions: (forecast_reference_time: 4, step: 2, level: 2,\n", + " latitude: 19, longitude: 36)\n", + "Coordinates:\n", + " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", + " * level (level) int64 16B 500 700\n", + " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", + " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", + "Data variables:\n", + " r (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", + " t (forecast_reference_time, step, level, latitude, longitude) float64 88kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import earthkit.data as ekd\n", + "ds_fl = ekd.from_source(\"sample\", \"pl.grib\").to_fieldlist()\n", + "ds = ds_fl.to_xarray()\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "1939ee80-8c1e-4498-8a80-d5e6ac65f8e7", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "#### Using remapping" + ] + }, + { + "cell_type": "markdown", + "id": "0921516f-74b2-417a-a85a-378525090348", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "Using the remapping option can come in handy when there are parameters with different level types in the input data." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f55fe598-7f80-4e9a-87b3-43131f78f2e1", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 1MB\n",
    +       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
    +       "                              latitude: 19, longitude: 36)\n",
    +       "Coordinates:\n",
    +       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
    +       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
    +       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
    +       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
    +       "Data variables: (12/32)\n",
    +       "    2t_0                     (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    msl_0                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_1000                   (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_300                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_400                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_500                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    ...                       ...\n",
    +       "    z_1000                   (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_300                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_400                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_500                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_700                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_850                    (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "Attributes:\n",
    +       "    Conventions:  CF-1.8\n",
    +       "    institution:  ECMWF
    " + ], + "text/plain": [ + " Size: 1MB\n", + "Dimensions: (forecast_reference_time: 4, step: 2,\n", + " latitude: 19, longitude: 36)\n", + "Coordinates:\n", + " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", + " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", + " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", + "Data variables: (12/32)\n", + " 2t_0 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " msl_0 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_1000 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_300 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_400 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " ... ...\n", + " z_1000 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_300 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_400 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_500 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_700 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_850 (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_sfc.grib\").to_fieldlist()\n", + "ds = ds_fl.to_xarray(variable_key=\"p_l\", remapping={\"p_l\": \"{parameter.variable}_{vertical.level}\"})\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "7e22e364-2518-4460-ab35-64230d49a703", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "We can take it one step further and define a metadata key that combines the param, the level and the level type into a single key." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e1bc4e80-2efc-460c-96b0-26553bed7591", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 1MB\n",
    +       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
    +       "                              latitude: 19, longitude: 36)\n",
    +       "Coordinates:\n",
    +       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
    +       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
    +       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
    +       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
    +       "Data variables: (12/32)\n",
    +       "    2t_0_surface             (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    msl_0_surface            (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_1000_pressure          (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_300_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_400_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    r_500_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    ...                       ...\n",
    +       "    z_1000_pressure          (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_300_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_400_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_500_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_700_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    z_850_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "Attributes:\n",
    +       "    Conventions:  CF-1.8\n",
    +       "    institution:  ECMWF
    " + ], + "text/plain": [ + " Size: 1MB\n", + "Dimensions: (forecast_reference_time: 4, step: 2,\n", + " latitude: 19, longitude: 36)\n", + "Coordinates:\n", + " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", + " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", + " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", + "Data variables: (12/32)\n", + " 2t_0_surface (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " msl_0_surface (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_1000_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_300_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_400_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " r_500_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " ... ...\n", + " z_1000_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_300_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_400_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_500_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_700_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " z_850_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_sfc.grib\").to_fieldlist()\n", + "ds = ds_fl.to_xarray(variable_key=\"p_l_t\", remapping={\"p_l_t\": \"{parameter.variable}_{vertical.level}_{vertical.level_type}\"})\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "374ea92f-5b3f-4134-adfb-1c93dc1258f7", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "This technique is particularly useful when the same parameter is available on multiple level types in the input data. In this case using \"param_level\" does not result in a full hypercube, however the same ``remapping`` that we used above does." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "39cbb360-43c4-416e-956b-8b6cfadda26c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 351kB\n",
    +       "Dimensions:                  (forecast_reference_time: 4, step: 2,\n",
    +       "                              latitude: 19, longitude: 36)\n",
    +       "Coordinates:\n",
    +       "  * forecast_reference_time  (forecast_reference_time) datetime64[ns] 32B 202...\n",
    +       "  * step                     (step) timedelta64[ns] 16B 00:00:00 06:00:00\n",
    +       "  * latitude                 (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n",
    +       "  * longitude                (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n",
    +       "Data variables:\n",
    +       "    t_137_hybrid             (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    t_500_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    t_700_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    t_90_hybrid              (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    u_137_hybrid             (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    u_500_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    u_700_pressure           (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "    u_90_hybrid              (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n",
    +       "Attributes:\n",
    +       "    Conventions:  CF-1.8\n",
    +       "    institution:  ECMWF
    " + ], + "text/plain": [ + " Size: 351kB\n", + "Dimensions: (forecast_reference_time: 4, step: 2,\n", + " latitude: 19, longitude: 36)\n", + "Coordinates:\n", + " * forecast_reference_time (forecast_reference_time) datetime64[ns] 32B 202...\n", + " * step (step) timedelta64[ns] 16B 00:00:00 06:00:00\n", + " * latitude (latitude) float64 152B 90.0 80.0 ... -80.0 -90.0\n", + " * longitude (longitude) float64 288B 0.0 10.0 ... 340.0 350.0\n", + "Data variables:\n", + " t_137_hybrid (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " t_500_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " t_700_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " t_90_hybrid (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " u_137_hybrid (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " u_500_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " u_700_pressure (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + " u_90_hybrid (forecast_reference_time, step, latitude, longitude) float64 44kB ...\n", + "Attributes:\n", + " Conventions: CF-1.8\n", + " institution: ECMWF" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fl = ekd.from_source(\"sample\", \"mixed_pl_ml.grib\").to_fieldlist()\n", + "ds = ds_fl.to_xarray(variable_key=\"p_l_t\", remapping={\"p_l_t\": \"{parameter.variable}_{vertical.level}_{vertical.level_type}\"})\n", + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4be3d1c2-873b-42a7-9a92-468d38ee64e2", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev", + "language": "python", + "name": "dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/earthkit/data/core/field.py b/src/earthkit/data/core/field.py index e1bdbf235..141b24bf3 100644 --- a/src/earthkit/data/core/field.py +++ b/src/earthkit/data/core/field.py @@ -51,14 +51,18 @@ METADATA = "metadata" DESCRIBE_ORDER = [ + DATA, PARAMETER, TIME, VERTICAL, ENSEMBLE, GEOGRAPHY, PROC, + LABELS, ] +DEFAULT_DESCRIBE_SELECTION = PARAMETER + class _ComponentMaker: """A factory class to delay import of component classes.""" @@ -1240,7 +1244,9 @@ def describe(self, component=all, filter=None, **kwargs): } ) - return format_namespace_dump(r, selected="parameter", details=self.__class__.__name__, **kwargs) + return format_namespace_dump( + r, selected=DEFAULT_DESCRIBE_SELECTION, details=self.__class__.__name__, **kwargs + ) @property def default_ls_keys(self): diff --git a/src/earthkit/data/data/bufr.py b/src/earthkit/data/data/bufr.py index e9eac5180..99290e8f1 100644 --- a/src/earthkit/data/data/bufr.py +++ b/src/earthkit/data/data/bufr.py @@ -28,3 +28,8 @@ def to_pandas(self, **kwargs): def to_featurelist(self, *args, **kwargs): return self._reader.to_featurelist(*args, **kwargs) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="BUFR file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/csv.py b/src/earthkit/data/data/csv.py index 9389230d3..2379ba7c6 100644 --- a/src/earthkit/data/data/csv.py +++ b/src/earthkit/data/data/csv.py @@ -25,3 +25,8 @@ def to_pandas(self, **kwargs): def to_xarray(self, **kwargs): return self._reader.to_xarray(**kwargs) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="CSV file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/geojson.py b/src/earthkit/data/data/geojson.py index 28f69d971..da2d2ef94 100644 --- a/src/earthkit/data/data/geojson.py +++ b/src/earthkit/data/data/geojson.py @@ -36,3 +36,8 @@ def to_featurelist(self, *args, **kwargs): from earthkit.data.readers.geojson.file import GeoJsonList return GeoJsonList(self._reader.path) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="GeoJSON file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/geotiff.py b/src/earthkit/data/data/geotiff.py index 0f37a52b5..1a6619604 100644 --- a/src/earthkit/data/data/geotiff.py +++ b/src/earthkit/data/data/geotiff.py @@ -28,3 +28,8 @@ def to_pandas(self, **kwargs): def to_xarray(self, **kwargs): return self._reader.to_xarray(**kwargs) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="GeoTIFF file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/grib.py b/src/earthkit/data/data/grib.py index f91ebce09..265d6d5f4 100644 --- a/src/earthkit/data/data/grib.py +++ b/src/earthkit/data/data/grib.py @@ -7,6 +7,7 @@ # nor does it submit to any jurisdiction. # + from .source import SourceData @@ -20,6 +21,9 @@ def available_types(self): def describe(self): return f"GRIB data from {self._reader.path}" + def __repr__(self): + return "GribData(path={self._reader.path})" + def to_fieldlist(self, *args, **kwargs): return self._reader.to_fieldlist(*args, **kwargs) @@ -34,3 +38,8 @@ def to_numpy(self, *args, **kwargs): def to_array(self, *args, **kwargs): return self._reader.to_array(*args, **kwargs) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="GRIB file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/netcdf.py b/src/earthkit/data/data/netcdf.py index 977c27dac..60f2e8aff 100644 --- a/src/earthkit/data/data/netcdf.py +++ b/src/earthkit/data/data/netcdf.py @@ -42,3 +42,8 @@ def to_array(self, *args, **kwargs): def _default_encoder(self): return self._source._default_encoder() + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="NetCDF file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/odb.py b/src/earthkit/data/data/odb.py index d502ee43e..48fb53697 100644 --- a/src/earthkit/data/data/odb.py +++ b/src/earthkit/data/data/odb.py @@ -31,3 +31,8 @@ def to_target(self, target, *args, **kwargs): from earthkit.data import to_target return to_target(target, *args, data=self._reader, **kwargs) + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="ODB file", path=self._reader.path, types=self.available_types) diff --git a/src/earthkit/data/data/stream.py b/src/earthkit/data/data/stream.py index 0e0ba3de2..f224c563a 100644 --- a/src/earthkit/data/data/stream.py +++ b/src/earthkit/data/data/stream.py @@ -53,3 +53,8 @@ def to_fieldlist(self, *args, read_all=False, **kwargs): return r return self._reader + + def _repr_html_(self): + from earthkit.data.utils.summary import make_data_repr_html + + return make_data_repr_html(title="Stream of fields", path=None, types=self.available_types) diff --git a/src/earthkit/data/encoders/zarr.py b/src/earthkit/data/encoders/zarr.py index 6ade88ff3..5aa53ef92 100644 --- a/src/earthkit/data/encoders/zarr.py +++ b/src/earthkit/data/encoders/zarr.py @@ -69,10 +69,10 @@ def _encode( ): return ZarrEncodedData(data.to_xarray(add_earthkit_attrs=False)) - def _encode_field(self, field, **kwargs): + def _encode_field(self, field, *, target=None, **kwargs): raise NotImplementedError("ZarrEncoder does not support encoding individual fields.") - def _encode_fieldlist(self, data, **kwargs): + def _encode_fieldlist(self, data, *, target=None, **kwargs): earthkit_to_xarray_kwargs = kwargs.pop("earthkit_to_xarray_kwargs", {}) # earthkit_to_xarray_kwargs.update(kwargs) earthkit_to_xarray_kwargs["add_earthkit_attrs"] = False @@ -81,11 +81,14 @@ def _encode_fieldlist(self, data, **kwargs): ds = data.to_xarray(**kwargs) return ZarrEncodedData(ds) - def _encode_xarray(self, data, **kwargs): + def _encode_xarray(self, data, *, target=None, **kwargs): raise NotImplementedError - def _encode_featurelist(self, data, **kwargs): + def _encode_featurelist(self, data, *, target=None, **kwargs): raise NotImplementedError + def _encode_path(self, path_info, *, target=None, **kwargs): + raise + encoder = ZarrEncoder diff --git a/src/earthkit/data/field/component/geography.py b/src/earthkit/data/field/component/geography.py index 535fb0c50..090595e9f 100644 --- a/src/earthkit/data/field/component/geography.py +++ b/src/earthkit/data/field/component/geography.py @@ -356,9 +356,6 @@ def grid_type(self) -> str: """ pass - def to_dict(self): - return {"grid_spec": self.grid_spec()} - # @classmethod # def from_dict(cls, data, shape_hint=None): # from ..dict.geography import create_geography @@ -456,6 +453,14 @@ def points(self, flatten=False, dtype=None): def from_dict(cls, data, shape_hint=None): return create_geography_from_dict(data, shape_hint=shape_hint) + def to_dict(self): + return { + "grid_spec": self.grid_spec(), + "grid_type": self.grid_type(), + "shape": self.shape(), + "area": self.area(), + } + def __getstate__(self): return super().__getstate__() @@ -566,16 +571,16 @@ def grid(self): return None def _north(self): - return np.amax(self.latitudes()) + return float(np.amax(self.latitudes())) def _south(self): - return np.amin(self.latitudes()) + return float(np.amin(self.latitudes())) - def west(self): - return np.amin(self.longitudes()) + def _west(self): + return float(np.amin(self.longitudes())) def _east(self): - return np.amax(self.longitudes()) + return float(np.amax(self.longitudes())) def projection(self): if self._proj_str: @@ -595,7 +600,7 @@ def grid_spec(self): return None def area(self) -> tuple: - return (self.north(), self.west(), self.south(), self.east()) + return (self._north(), self._west(), self._south(), self._east()) def grid_type(self): return "_unstructured" diff --git a/src/earthkit/data/field/grib/geography.py b/src/earthkit/data/field/grib/geography.py index bbec5a52a..2ca559cbc 100644 --- a/src/earthkit/data/field/grib/geography.py +++ b/src/earthkit/data/field/grib/geography.py @@ -152,8 +152,8 @@ def grid_type(self): def from_dict(*args, **kwargs): raise NotImplementedError("GribGeography cannot be created from a dictionary") - def to_dict(self): - return dict() + # def to_dict(self): + # return dict() def __getstate__(self): state = {} diff --git a/src/earthkit/data/field/handler/data.py b/src/earthkit/data/field/handler/data.py index e77ecfac0..ca6fdcdc1 100644 --- a/src/earthkit/data/field/handler/data.py +++ b/src/earthkit/data/field/handler/data.py @@ -73,6 +73,17 @@ def get(self, key, default=None, *, astype=None, raise_on_missing=False): return default + def to_dict(self): + values = self.get_values(copy=False) + if values is None: + return dict() + else: + return { + "number_of_values": values.size, + "array_type": type(values).__name__, + "array_dtype": str(values.dtype), + } + class DataFieldComponentHandler(BaseDataFieldComponentHandler): """Data component handler for Field that provides implementation @@ -156,9 +167,15 @@ def set(self, values=None, **kwargs): return self.set_values(values) raise ValueError("Invalid arguments") - def dump(self, *args, **kwargs): - """This field component has no dump.""" - return None + def dump(self, owner: Any, name: str, result: dict, prefix_keys=False) -> None: + """Populate the namespace dictionary for this SpecFieldComponent.""" + + def _prefix(key): + return f"{self.NAME}.{key}" if prefix_keys else key + + if name is None or name == self.NAME or (isinstance(name, (list, tuple)) and self.NAME in name): + r = {_prefix(k): v for k, v in self.to_dict().items()} + result[self.NAME] = r def check(self, owner): """Check that the data is consistent with the field's shape.""" diff --git a/src/earthkit/data/readers/bufr/scan.py b/src/earthkit/data/readers/bufr/scan.py index 1dbc7f1de..481d3d22f 100644 --- a/src/earthkit/data/readers/bufr/scan.py +++ b/src/earthkit/data/readers/bufr/scan.py @@ -16,7 +16,7 @@ class BufrCodesMessagePositionIndex(CodesMessagePositionIndex): MAGIC = b"BUFR" # This does not belong here, should be in the C library - def _get_message_positions_part(self, fd, part): + def _get_message_positions_part(self, fd, part, max_count=None): assert part is not None assert len(part) == 2 @@ -26,6 +26,7 @@ def _get_message_positions_part(self, fd, part): if os.lseek(fd, offset, os.SEEK_SET) != offset: return + count = 0 while True: code = os.read(fd, 4) if len(code) < 4: @@ -43,5 +44,9 @@ def _get_message_positions_part(self, fd, part): if edition in [3, 4]: yield offset, length + count += 1 + + if max_count is not None and count >= max_count: + return offset = os.lseek(fd, offset + length, os.SEEK_SET) diff --git a/src/earthkit/data/readers/grib/file.py b/src/earthkit/data/readers/grib/file.py index ec68b0c8b..725ef0f79 100644 --- a/src/earthkit/data/readers/grib/file.py +++ b/src/earthkit/data/readers/grib/file.py @@ -17,6 +17,19 @@ from .scan import GribCodesMessagePositionIndex +def first_field_from_grib_file(path, parts=None): + offset = GribCodesMessagePositionIndex(path, parts=parts, max_count=1).offsets[0] + if offset < 0 or offset is None: + return None + from .memory import GribStreamReader + + with open(path, "rb") as f: + if parts: + f.seek(offset) + reader = GribStreamReader(f) + return next(reader, None) + + class GribFieldListInFile(SimpleFieldListBase, GRIBReaderBase): handle_cache = None @@ -192,6 +205,9 @@ def to_numpy(self, *args, **kwargs): def to_array(self, *args, **kwargs): return self.to_fieldlist().to_array(*args, **kwargs) + def peek(self): + return first_field_from_grib_file(self.path, parts=self._kwargs.get("parts", None)) + def mutate_source(self): # A GRIBReader is a source itself return self diff --git a/src/earthkit/data/readers/grib/scan.py b/src/earthkit/data/readers/grib/scan.py index f9dee2bfe..45f108c79 100644 --- a/src/earthkit/data/readers/grib/scan.py +++ b/src/earthkit/data/readers/grib/scan.py @@ -19,7 +19,7 @@ class GribCodesMessagePositionIndex(CodesMessagePositionIndex): MAGIC = b"GRIB" # This does not belong here, should be in the C library - def _get_message_positions_part(self, fd, part): + def _get_message_positions_part(self, fd, part, max_count=None): assert part is not None assert len(part) == 2 @@ -29,6 +29,7 @@ def _get_message_positions_part(self, fd, part): if os.lseek(fd, offset, os.SEEK_SET) != offset: return + count = 0 while True: code = os.read(fd, 4) if len(code) < 4: @@ -71,4 +72,10 @@ def _get_message_positions_part(self, fd, part): return yield offset, length + + if max_count is not None: + count += 1 + if count >= max_count: + return + offset = os.lseek(fd, offset + length, os.SEEK_SET) diff --git a/src/earthkit/data/utils/message.py b/src/earthkit/data/utils/message.py index a711074c5..750b153d7 100644 --- a/src/earthkit/data/utils/message.py +++ b/src/earthkit/data/utils/message.py @@ -83,11 +83,12 @@ class CodesMessagePositionIndex: VERSION = 1 MAGIC = None - def __init__(self, path, parts=None): + def __init__(self, path, parts=None, max_count=None): self.path = path self.offsets = None self.lengths = None self.parts = parts + self.max_count = max_count self._cache_file = None self._load() @@ -99,11 +100,11 @@ def _get_message_positions(self, path, parts): try: if parts is None: - yield from self._get_message_positions_part(fd, (0, -1)) + yield from self._get_message_positions_part(fd, (0, -1), max_count=self.max_count) else: for part in parts: try: - yield from self._get_message_positions_part(fd, part) + yield from self._get_message_positions_part(fd, part, max_count=self.max_count) except Exception: pass except Exception: @@ -111,7 +112,7 @@ def _get_message_positions(self, path, parts): finally: os.close(fd) - def _get_message_positions_part(self, path, part): + def _get_message_positions_part(self, path, part, max_count=None): raise NotImplementedError @staticmethod diff --git a/src/earthkit/data/utils/summary.py b/src/earthkit/data/utils/summary.py index c189d4f76..a964c8523 100644 --- a/src/earthkit/data/utils/summary.py +++ b/src/earthkit/data/utils/summary.py @@ -425,3 +425,17 @@ def ncdump(path): t = result.stdout.decode("utf-8").split("\n") for line in t: print(line) + + +def make_data_repr_html(title=None, path=None, types=None): + import os + + from earthkit.data.utils.humanize import bytes + + t = f"

    {title}
    " + if path is not None: + t += f"Path: {path} size: {bytes(os.path.getsize(path))}
    " + + t += f"Available types: {', '.join(types)}

    " + + return t