diff --git a/data/notebooks/pipes_mock/tiles.ipynb b/data/notebooks/pipes_mock/tiles.ipynb index 96a7a6c4..de2a8172 100644 --- a/data/notebooks/pipes_mock/tiles.ipynb +++ b/data/notebooks/pipes_mock/tiles.ipynb @@ -2,9 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload\n", "%autoreload 2" @@ -12,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -37,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 89, "metadata": {}, "outputs": [ { @@ -67,44 +76,98 @@ "/home/mambauser/data/eez/processed/eez_preprocess.zip\n", "/home/mambauser/data/eez/processed/preprocess\n" ] - }, + } + ], + "source": [ + "pipe = \"eez\"\n", + "collection_name = f\"{pipe}_v11\"\n", + "\n", + "eez_dir = FileConventionHandler(pipe)\n", + "\n", + "# Download the EEZ file && unzip it\n", + "download_and_unzip_if_needed(eez_dir, prev_step, mysettings)" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [], + "source": [ + "# Download the translations file\n", + "input_file = eez_dir.get_processed_step_path(prev_step).joinpath(\"eez_translation.csv\")\n", + "\n", + "writeReadGCP(\n", + " credentials=mysettings.GCS_KEYFILE_JSON,\n", + " bucket_name=mysettings.GCS_BUCKET,\n", + " blob_name=\"vizzuality_processed_data/eez/preprocess/eez_translation.csv\",\n", + " file=input_file,\n", + " operation=\"r\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Allocating 8 GB of heap memory\n", - "[o] Wrote /home/mambauser/data/eez/processed/tiles/eez_tiles.json\n" + "/opt/conda/lib/python3.12/site-packages/pyogrio/raw.py:709: RuntimeWarning: Value 212881389 of field AREA_KM2 of feature 281 not successfully written. Possibly due to too larger number with respect to field width\n", + " ogr_write(\n" + ] + } + ], + "source": [ + "# Add translations\n", + "eez = gpd.read_file(eez_dir.get_step_fmt_file_path(prev_step, \"shp\").as_posix())\n", + "translations = pd.read_csv(input_file).drop(columns=[\"GEONAME\", \"POL_TYPE\"])\n", + "eez_translation = eez.merge(translations, on=\"MRGID\", how=\"left\")\n", + "eez_translation.to_file(eez_dir.get_processed_step_path(prev_step).joinpath(\"eez_translation.shp\").as_posix())" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "For layer 0, using name \"eez_tiles\"\n", + "/home/mambauser/data/eez/processed/tiles/eez_tiles.json:10: Found ] at top level\n", + "/home/mambauser/data/eez/processed/tiles/eez_tiles.json:55: Reached EOF without all containers being closed\n", + "In JSON object {\"type\":\"FeatureCollection\",\"features\":[]}\n", + "282 features, 62937109 bytes of geometry, 7882 bytes of separate metadata, 38990 bytes of string pool\n", + "Choosing a maxzoom of -z0 for features about 2511227 feet (765422 meters) apart\n", + "Choosing a maxzoom of -z9 for resolution of about 825 feet (251 meters) within features\n", + " 99.9% 9/136/131 \n" ] }, { "data": { "text/plain": [ - "PosixPath('/home/mambauser/data/eez/processed/tiles/eez_v11.mbtiles')" + "PosixPath('/home/mambauser/data/eez/processed/tiles/eez_tiles.mbtiles')" ] }, - "execution_count": 5, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pipe = \"eez\"\n", - "collection_name = f\"{pipe}_v11\"\n", - "\n", - "eez_dir = FileConventionHandler(pipe)\n", - "# Download the EEZ file && unzip it\n", - "download_and_unzip_if_needed(eez_dir, prev_step, mysettings)\n", - "\n", - "# simplify the geometries\n", - "Mapshaper(8).input([eez_dir.get_step_fmt_file_path(prev_step, \"shp\").as_posix()]).filter_fields(\n", - " fields=\",\".join([\"GEONAME\", \"POL_TYPE\", \"ISO_SOV1\", \"ISO_SOV2\", \"ISO_SOV3\"])\n", + "# Simplify the geometries\n", + "Mapshaper(8).input([eez_dir.get_processed_step_path(prev_step).joinpath(\"eez_translation.shp\").as_posix()]).filter_fields(\n", + " fields=\",\".join([\"GEONAME\", \"GEONAME_ES\", \"GEONAME_FR\", \"POL_TYPE\", \"ISO_SOV1\", \"ISO_SOV2\", \"ISO_SOV3\"])\n", ").output(\n", " eez_dir.get_step_fmt_file_path(current_step, \"json\").as_posix(), force=True, format=\"geojson\"\n", ").execute()\n", - "mbtileGeneration(\n", - " eez_dir.get_step_fmt_file_path(current_step, \"json\"),\n", - " eez_dir.get_processed_step_path(current_step).joinpath(f\"{collection_name}.mbtiles\"),\n", - ")" + "\n", + "# Generate mbtiles\n", + "mbtileGeneration(eez_dir.get_step_fmt_file_path(current_step, \"json\"))" ] }, { @@ -233,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -245,25 +308,25 @@ ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "Allocating 16 GB of heap memory\n", - "[clean] Retained 19,184 of 19,184 features\n", - "[simplify] Repaired 277,407 intersections; 443,678 intersections could not be repaired\n", - "[clean] Retained 19,184 of 19,184 features\n", - "[o] Wrote /home/mambauser/data/mpa/processed/tiles/mpa_tiles.json\n" + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[61], line 7\u001b[0m\n\u001b[1;32m 4\u001b[0m source_dir \u001b[38;5;241m=\u001b[39m FileConventionHandler(pipe)\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# Download the EEZ file && unzip it\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m \u001b[43mdownload_and_unzip_if_needed\u001b[49m\u001b[43m(\u001b[49m\u001b[43msource_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprev_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmysettings\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;66;03m# simplify the geometries\u001b[39;00m\n\u001b[1;32m 10\u001b[0m Mapshaper(\u001b[38;5;241m16\u001b[39m)\u001b[38;5;241m.\u001b[39minput([source_dir\u001b[38;5;241m.\u001b[39mget_step_fmt_file_path(prev_step, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshp\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mas_posix()])\u001b[38;5;241m.\u001b[39mfilter_fields(\n\u001b[1;32m 11\u001b[0m fields\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWDPAID,NAME,PA_DEF,GIS_M_AREA,PARENT_ISO\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 12\u001b[0m )\u001b[38;5;241m.\u001b[39mclean(allow_overlaps\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, rewind\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\u001b[38;5;241m.\u001b[39msimplify(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdp 10\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m keep-shapes planar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mclean(\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 15\u001b[0m source_dir\u001b[38;5;241m.\u001b[39mget_step_fmt_file_path(current_step, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mjson\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mas_posix(), force\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeojson\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 16\u001b[0m )\u001b[38;5;241m.\u001b[39mexecute()\n", + "File \u001b[0;32m~/src/helpers/utils.py:140\u001b[0m, in \u001b[0;36mdownload_and_unzip_if_needed\u001b[0;34m(file_handler, prev_step, mysettings)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m unzzipped_path\n\u001b[1;32m 139\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m zip_path\u001b[38;5;241m.\u001b[39mexists():\n\u001b[0;32m--> 140\u001b[0m \u001b[43mwriteReadGCP\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 141\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmysettings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGCS_KEYFILE_JSON\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 142\u001b[0m \u001b[43m \u001b[49m\u001b[43mbucket_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmysettings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGCS_BUCKET\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 143\u001b[0m \u001b[43m \u001b[49m\u001b[43mblob_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfile_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_remote_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprev_step\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 144\u001b[0m \u001b[43m \u001b[49m\u001b[43mfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzip_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 145\u001b[0m \u001b[43m \u001b[49m\u001b[43moperation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 146\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 148\u001b[0m shutil\u001b[38;5;241m.\u001b[39munpack_archive(zip_path, unzzipped_path\u001b[38;5;241m.\u001b[39mparent)\n\u001b[1;32m 150\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m unzzipped_path\n", + "File \u001b[0;32m~/src/helpers/utils.py:108\u001b[0m, in \u001b[0;36mwriteReadGCP\u001b[0;34m(credentials, bucket_name, blob_name, file, operation)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m operation \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(file, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 108\u001b[0m \u001b[43mblob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_to_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperation must be \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m or \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/contextlib.py:81\u001b[0m, in \u001b[0;36mContextDecorator.__call__..inner\u001b[0;34m(*args, **kwds)\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(func)\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minner\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds):\n\u001b[1;32m 80\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_recreate_cm():\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/cloud/storage/blob.py:1192\u001b[0m, in \u001b[0;36mBlob.download_to_file\u001b[0;34m(self, file_obj, client, start, end, raw_download, if_etag_match, if_etag_not_match, if_generation_match, if_generation_not_match, if_metageneration_match, if_metageneration_not_match, timeout, checksum, retry)\u001b[0m\n\u001b[1;32m 1073\u001b[0m \u001b[38;5;129m@create_trace_span\u001b[39m(name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStorage.Blob.downloadToFile\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1074\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdownload_to_file\u001b[39m(\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1089\u001b[0m retry\u001b[38;5;241m=\u001b[39mDEFAULT_RETRY,\n\u001b[1;32m 1090\u001b[0m ):\n\u001b[1;32m 1091\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Download the contents of this blob into a file-like object.\u001b[39;00m\n\u001b[1;32m 1092\u001b[0m \n\u001b[1;32m 1093\u001b[0m \u001b[38;5;124;03m .. note::\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1189\u001b[0m \u001b[38;5;124;03m :raises: :class:`google.cloud.exceptions.NotFound`\u001b[39;00m\n\u001b[1;32m 1190\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1192\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_prep_and_do_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1193\u001b[0m \u001b[43m \u001b[49m\u001b[43mfile_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1194\u001b[0m \u001b[43m \u001b[49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1195\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1196\u001b[0m \u001b[43m \u001b[49m\u001b[43mend\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1197\u001b[0m \u001b[43m \u001b[49m\u001b[43mraw_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraw_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1198\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_etag_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_etag_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1199\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_etag_not_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_etag_not_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1200\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_generation_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_generation_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1201\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_generation_not_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_generation_not_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1202\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_metageneration_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_metageneration_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1203\u001b[0m \u001b[43m \u001b[49m\u001b[43mif_metageneration_not_match\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mif_metageneration_not_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1204\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1205\u001b[0m \u001b[43m \u001b[49m\u001b[43mchecksum\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchecksum\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1206\u001b[0m \u001b[43m \u001b[49m\u001b[43mretry\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretry\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1207\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/cloud/storage/blob.py:4420\u001b[0m, in \u001b[0;36mBlob._prep_and_do_download\u001b[0;34m(self, file_obj, client, start, end, raw_download, if_etag_match, if_etag_not_match, if_generation_match, if_generation_not_match, if_metageneration_match, if_metageneration_not_match, timeout, checksum, retry, command)\u001b[0m\n\u001b[1;32m 4417\u001b[0m transport \u001b[38;5;241m=\u001b[39m client\u001b[38;5;241m.\u001b[39m_http\n\u001b[1;32m 4419\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 4420\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 4421\u001b[0m \u001b[43m \u001b[49m\u001b[43mtransport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4422\u001b[0m \u001b[43m \u001b[49m\u001b[43mfile_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4423\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4424\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4425\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4426\u001b[0m \u001b[43m \u001b[49m\u001b[43mend\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4427\u001b[0m \u001b[43m \u001b[49m\u001b[43mraw_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4428\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4429\u001b[0m \u001b[43m \u001b[49m\u001b[43mchecksum\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchecksum\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4430\u001b[0m \u001b[43m \u001b[49m\u001b[43mretry\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretry\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4431\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4432\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m resumable_media\u001b[38;5;241m.\u001b[39mInvalidResponse \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 4433\u001b[0m _raise_from_invalid_response(exc)\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/cloud/storage/blob.py:1041\u001b[0m, in \u001b[0;36mBlob._do_download\u001b[0;34m(self, transport, file_obj, download_url, headers, start, end, raw_download, timeout, checksum, retry)\u001b[0m\n\u001b[1;32m 1035\u001b[0m download\u001b[38;5;241m.\u001b[39m_retry_strategy \u001b[38;5;241m=\u001b[39m retry_strategy\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m create_trace_span(\n\u001b[1;32m 1037\u001b[0m name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStorage.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdownload_class\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/consume\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1038\u001b[0m attributes\u001b[38;5;241m=\u001b[39mextra_attributes,\n\u001b[1;32m 1039\u001b[0m api_request\u001b[38;5;241m=\u001b[39margs,\n\u001b[1;32m 1040\u001b[0m ):\n\u001b[0;32m-> 1041\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mdownload\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconsume\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransport\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1042\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_extract_headers_from_download(response)\n\u001b[1;32m 1043\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/resumable_media/requests/download.py:263\u001b[0m, in \u001b[0;36mDownload.consume\u001b[0;34m(self, transport, timeout)\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_write_to_stream(result)\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[0;32m--> 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_request_helpers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_and_retry\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 264\u001b[0m \u001b[43m \u001b[49m\u001b[43mretriable_request\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_status_code\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_strategy\u001b[49m\n\u001b[1;32m 265\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/resumable_media/requests/_request_helpers.py:155\u001b[0m, in \u001b[0;36mwait_and_retry\u001b[0;34m(func, get_status_code, retry_strategy)\u001b[0m\n\u001b[1;32m 153\u001b[0m error \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 154\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 155\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _CONNECTION_ERROR_CLASSES \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 157\u001b[0m error \u001b[38;5;241m=\u001b[39m e \u001b[38;5;66;03m# Fall through to retry, if there are retries left.\u001b[39;00m\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/resumable_media/requests/download.py:259\u001b[0m, in \u001b[0;36mDownload.consume..retriable_request\u001b[0;34m()\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bytes_downloaded \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m--> 259\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_write_to_stream\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresult\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n", + "File \u001b[0;32m/opt/conda/lib/python3.12/site-packages/google/resumable_media/requests/download.py:134\u001b[0m, in \u001b[0;36mDownload._write_to_stream\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stream\u001b[38;5;241m.\u001b[39mwrite(chunk)\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bytes_downloaded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(chunk)\n\u001b[0;32m--> 134\u001b[0m \u001b[43mlocal_checksum_object\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupdate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunk\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;66;03m# Don't validate the checksum for partial responses.\u001b[39;00m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 138\u001b[0m expected_checksum \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m!=\u001b[39m http\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mPARTIAL_CONTENT\n\u001b[1;32m 140\u001b[0m ):\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] - }, - { - "data": { - "text/plain": [ - "PosixPath('/home/mambauser/data/mpa/processed/tiles/mpa_tiles.mbtiles')" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -290,7 +353,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ {