From cacc96c7ca3c31e560455f46e12e443dd4c05005 Mon Sep 17 00:00:00 2001 From: Alicia Date: Thu, 26 Oct 2023 09:13:41 +0200 Subject: [PATCH 1/2] Strapi connection --- cms/docker-compose.yml | 19 ++- .../tiles_generation/EEZ_tiles.ipynb | 157 ++++++++++++++++++ data/src/cleaning_df.py | 0 data/src/strapi.py | 93 +++++++++++ 4 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 data/notebooks/tiles_generation/EEZ_tiles.ipynb create mode 100644 data/src/cleaning_df.py create mode 100644 data/src/strapi.py diff --git a/cms/docker-compose.yml b/cms/docker-compose.yml index 673cd6d9..2311b889 100644 --- a/cms/docker-compose.yml +++ b/cms/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.9" # optional since v1.27.0 +version: "3.9" # optional since v1.27.0 services: cms: build: . @@ -6,3 +6,20 @@ services: - "1337:1337" extra_hosts: - "host.docker.internal:host-gateway" + env_file: + - .env + depends_on: + - db + + db: + image: postgres + restart: always + env_file: + - .env + ports: + - "5432:5432" + volumes: + - db-data:/var/lib/postgresql/data + +volumes: + db-data: diff --git a/data/notebooks/tiles_generation/EEZ_tiles.ipynb b/data/notebooks/tiles_generation/EEZ_tiles.ipynb new file mode 100644 index 00000000..09816d0b --- /dev/null +++ b/data/notebooks/tiles_generation/EEZ_tiles.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "79966c59", + "metadata": {}, + "source": [ + "This notebook relies on the following cmd line tools, ensure they are installed and in the system\n", + "- tippecanoe\n", + "- mapshaper\n", + "- aws cli" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "def1ef65-1fd6-4367-a620-2dbe415f7104", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "import logging\n", + "\n", + "from IPython.lib import backgroundjobs as bg\n", + "\n", + "scripts_dir = Path('../..').joinpath('src')\n", + "if scripts_dir not in sys.path:\n", + " sys.path.insert(0, scripts_dir.resolve().as_posix())\n", + " \n", + "from pipelines.pipes import get_pipes\n", + "from strapi import Strapi\n", + "\n", + "logging.basicConfig(level=logging.DEBUG)\n", + "logging.getLogger(\"requests\").setLevel(logging.WARNING)\n", + "logging.getLogger(\"urllib3\").setLevel(logging.WARNING)\n", + "jobs = bg.BackgroundJobManager()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "eb61ff00", + "metadata": {}, + "outputs": [ + { + "ename": "HTTPError", + "evalue": "404 Client Error: Not Found for url: http://192.168.50.116:1337/content-manager/collection-types", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/alicitita/Projects/skytruth-30x30/data/notebooks/tiles_generation/EEZ_tiles.ipynb Cell 3\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 1\u001b[0m strapi \u001b[39m=\u001b[39m Strapi(url\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mhttp://192.168.50.116:1337\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 2\u001b[0m strapi\u001b[39m.\u001b[39mlogin(\u001b[39m'\u001b[39m\u001b[39malicia\u001b[39m\u001b[39m'\u001b[39m,\u001b[39m'\u001b[39m\u001b[39mKitiara_90\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 3\u001b[0m )\n\u001b[0;32m----> 5\u001b[0m strapi\u001b[39m.\u001b[39;49mgetCollections\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/functools.py:981\u001b[0m, in \u001b[0;36mcached_property.__get__\u001b[0;34m(self, instance, owner)\u001b[0m\n\u001b[1;32m 979\u001b[0m val \u001b[39m=\u001b[39m cache\u001b[39m.\u001b[39mget(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mattrname, _NOT_FOUND)\n\u001b[1;32m 980\u001b[0m \u001b[39mif\u001b[39;00m val \u001b[39mis\u001b[39;00m _NOT_FOUND:\n\u001b[0;32m--> 981\u001b[0m val \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfunc(instance)\n\u001b[1;32m 982\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 983\u001b[0m cache[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mattrname] \u001b[39m=\u001b[39m val\n", + "File \u001b[0;32m~/src/strapi.py:53\u001b[0m, in \u001b[0;36mStrapi.getCollections\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[39m@cached_property\u001b[39m\n\u001b[1;32m 51\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgetCollections\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 52\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msession\u001b[39m.\u001b[39mget(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m/content-manager/collection-types\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m---> 53\u001b[0m response\u001b[39m.\u001b[39;49mraise_for_status()\n\u001b[1;32m 54\u001b[0m data \u001b[39m=\u001b[39m [\n\u001b[1;32m 55\u001b[0m response\u001b[39m.\u001b[39mjson()\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 56\u001b[0m \u001b[39mfor\u001b[39;00m collection \u001b[39min\u001b[39;00m response\u001b[39m.\u001b[39mjson()\n\u001b[1;32m 57\u001b[0m \u001b[39mif\u001b[39;00m collection\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39misDisplayed\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m==\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ]\n\u001b[1;32m 59\u001b[0m \u001b[39mreturn\u001b[39;00m data\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/requests/models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1016\u001b[0m http_error_msg \u001b[39m=\u001b[39m (\n\u001b[1;32m 1017\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstatus_code\u001b[39m}\u001b[39;00m\u001b[39m Server Error: \u001b[39m\u001b[39m{\u001b[39;00mreason\u001b[39m}\u001b[39;00m\u001b[39m for url: \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1018\u001b[0m )\n\u001b[1;32m 1020\u001b[0m \u001b[39mif\u001b[39;00m http_error_msg:\n\u001b[0;32m-> 1021\u001b[0m \u001b[39mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m)\n", + "\u001b[0;31mHTTPError\u001b[0m: 404 Client Error: Not Found for url: http://192.168.50.116:1337/content-manager/collection-types" + ] + } + ], + "source": [ + "strapi = Strapi(url='http://192.168.50.116:1337')\n", + "strapi.login()\n", + "\n", + "strapi.getCollections" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c4b3fc7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:pipelines.settings:/home/mambauser/data\n", + "DEBUG:root:Starting job # 0 in a separate thread.\n", + "INFO:pipelines.base_pipe:Running eez_tiles pipeline\n", + "INFO:utils:File /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118.zip already exists.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:utils:Unzip Finish.\n", + "Allocating 8 GB of heap memory\n", + "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.shp\n", + "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.shx\n", + "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.dbf\n", + "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.prj\n", + "INFO:mapbox_uploader:Uploading to Mapbox...\n", + "INFO:mapbox_uploader:Uploading to S3...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 69.8 MiB/69.8 MiB (7.8 MiB/s) with 1 file(s) remaining \r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:mapbox_uploader:CompletedProcess(args='aws s3 cp /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.mbtiles s3://tilestream-tilesets-production/53/_pending/h75tk8ezapo2qst37s0l04olc/skytruth --region us-east-1', returncode=0)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "upload: ../../data/eez_tiles/World_EEZ_v11_20191118/eez_v11.mbtiles to s3://tilestream-tilesets-production/53/_pending/h75tk8ezapo2qst37s0l04olc/skytruth\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [01:48<00:00, 1.08s/it]\n", + "INFO:mapbox_uploader:True\n" + ] + } + ], + "source": [ + "# this code will execute all pipelines selected in background.\n", + "mypipes_subset = {k: v for k, v in get_pipes().items() if k in ['eez_tiles']}\n", + "for name, pipe in mypipes_subset.items():\n", + " jobs.new(pipe().execute)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/data/src/cleaning_df.py b/data/src/cleaning_df.py new file mode 100644 index 00000000..e69de29b diff --git a/data/src/strapi.py b/data/src/strapi.py new file mode 100644 index 00000000..e369bdb2 --- /dev/null +++ b/data/src/strapi.py @@ -0,0 +1,93 @@ +from typing import Union +from requests import Session +from getpass import getpass +from pathlib import Path +from functools import cached_property +import logging + +logger = logging.getLogger(__name__) + + +class Strapi: + session: Union[str, None] = None + url: Union[str, None] = None + + def __init__(self, url): + self.url = url + self.session = Session() + + def login( + self, + username: Union[str, None] = None, + password: Union[str, None] = None, + jwt: Union[str, None] = None, + ): + """Login to Strapi and set the JWT token in the session headers + Remember that strapi has 3 types of JWT tokens: + - API tokens: for public access + - Authenticated user tokens: for authenticated users + - Admin tokens: for admin users + + only authenticated user tokens are valid for the content manager get Collections. + """ + if not jwt: + if not username: + username = getpass("Enter your username: ") + if not password: + password = getpass("Enter your password: ") + + try: + r = self.session.post( + f"{self.url}/api/auth/local", + json={"identifier": username, "password": password}, + headers={"accept": "application/json"}, + ) + r.raise_for_status() + jwt = r.json().get("jwt") + except Exception as e: + logger.error(r.json()) + raise ValueError(r.json().get("error")) + + self.session.headers.update({"Authorization": f"Bearer {jwt}"}) + + return self + + def logout(self) -> None: + self.session.close() + + @cached_property + def getCollections(self): + response = self.session.get(f"{self.url}/content-manager/collection-types") + response.raise_for_status() + data = [ + response.json().get("data") + for collection in response.json() + if collection.get("isDisplayed") == True + ] + return data + + def getCollectionMetadata(self, collectionName): + return list(filter(lambda x: (collectionName in x), self.getCollections)) + + def getCollectionData(self, collectionName): + response = self.session.get(f"{self.url}/api/{collectionName}") + response.raise_for_status() + return response.json() + + def importCollection(self, collectionName: str, file_path: Path, idField: str): + extension = file_path.suffix + + with open(file_path, "rb") as f: + data = f.read() + + response = self.session.post( + f"{self.url}/api/import-export-entries/content/import", + json={ + "idField": idField, + "slug": f"api::{collectionName}.{collectionName}", + "data": data, + "format": extension, + }, + ) + response.raise_for_status() + return self From f0ed01d6b9592d3eaee88bad17248b8fc3c02f73 Mon Sep 17 00:00:00 2001 From: Alicia Date: Thu, 2 Nov 2023 14:06:40 +0100 Subject: [PATCH 2/2] updated strapi class --- data/notebooks/strapi_connection.ipynb | 957 ++++++++++++++++++ .../tiles_generation/EEZ_tiles.ipynb | 157 --- data/src/strapi.py | 32 +- 3 files changed, 977 insertions(+), 169 deletions(-) create mode 100644 data/notebooks/strapi_connection.ipynb delete mode 100644 data/notebooks/tiles_generation/EEZ_tiles.ipynb diff --git a/data/notebooks/strapi_connection.ipynb b/data/notebooks/strapi_connection.ipynb new file mode 100644 index 00000000..b30468be --- /dev/null +++ b/data/notebooks/strapi_connection.ipynb @@ -0,0 +1,957 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "79966c59", + "metadata": {}, + "source": [ + "This notebook example the usage of the strapi connector to upload the data. \n", + "In order to make it work properly you will need a strapi user with reading permision for the collections, for the collection types, and for the import-export pluggin " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "def1ef65-1fd6-4367-a620-2dbe415f7104", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import sys\n", + "import logging\n", + "\n", + "scripts_dir = Path('..').joinpath('src')\n", + "if scripts_dir not in sys.path:\n", + " sys.path.insert(0, scripts_dir.resolve().as_posix())\n", + "\n", + "from strapi import Strapi\n", + "\n", + "logging.basicConfig(level=logging.DEBUG)\n", + "logging.getLogger(\"requests\").setLevel(logging.WARNING)\n", + "logging.getLogger(\"urllib3\").setLevel(logging.WARNING)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "eb61ff00", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'uid': 'api::data-info.data-info',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'data-info',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'data-info',\n", + " 'pluralName': 'data-infos',\n", + " 'displayName': 'Data Info',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'content': {'type': 'text', 'required': False},\n", + " 'data_sources': {'type': 'relation',\n", + " 'relation': 'oneToMany',\n", + " 'target': 'api::data-source.data-source',\n", + " 'targetModel': 'api::data-source.data-source',\n", + " 'relationType': 'oneToMany'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::data-source.data-source',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'data-source',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'data-source',\n", + " 'pluralName': 'data-sources',\n", + " 'displayName': 'Data Source',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'title': {'type': 'string', 'required': True},\n", + " 'url': {'type': 'string'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::fishing-protection-level.fishing-protection-level',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'fishing-protection-level',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'fishing-protection-level',\n", + " 'pluralName': 'fishing-protection-levels',\n", + " 'displayName': 'Fishing Protection Level'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'info': {'type': 'text'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::fishing-protection-level-stat.fishing-protection-level-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'fishing-protection-level-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'fishing-protection-level-stat',\n", + " 'pluralName': 'fishing-protection-level-stats',\n", + " 'displayName': 'Fishing Protection Level Stats',\n", + " 'description': 'Calculation of area of protection by location and fishing protection level'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'manyToOne',\n", + " 'target': 'api::location.location',\n", + " 'inversedBy': 'fishing_protection_level_stats',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'manyToOne'},\n", + " 'fishing_protection_level': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::fishing-protection-level.fishing-protection-level',\n", + " 'targetModel': 'api::fishing-protection-level.fishing-protection-level',\n", + " 'relationType': 'oneToOne'},\n", + " 'area': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::habitat.habitat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'habitat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'habitat',\n", + " 'pluralName': 'habitats',\n", + " 'displayName': 'Habitat',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'info': {'type': 'text'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::habitat-stat.habitat-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'habitat-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'habitat-stat',\n", + " 'pluralName': 'habitat-stats',\n", + " 'displayName': 'Habitat Stats',\n", + " 'description': 'Calculation of area of protection by location, habitat and year'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::location.location',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'oneToOne'},\n", + " 'habitat': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::habitat.habitat',\n", + " 'targetModel': 'api::habitat.habitat',\n", + " 'relationType': 'oneToOne'},\n", + " 'year': {'type': 'integer', 'required': True},\n", + " 'protectedArea': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'totalArea': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::layer.layer',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'layer',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'layer',\n", + " 'pluralName': 'layers',\n", + " 'displayName': 'Layer'},\n", + " 'options': {'draftAndPublish': True},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'title': {'type': 'string', 'required': True},\n", + " 'type': {'type': 'enumeration', 'enum': ['mapbox', 'deckgl', 'carto']},\n", + " 'config': {'type': 'json', 'required': True},\n", + " 'params_config': {'type': 'json', 'required': True},\n", + " 'legend_config': {'type': 'json', 'required': True},\n", + " 'interaction_config': {'type': 'json'},\n", + " 'metadata': {'type': 'component',\n", + " 'repeatable': False,\n", + " 'component': 'documentation.metadata'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::location.location',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'location',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'location',\n", + " 'pluralName': 'locations',\n", + " 'displayName': 'Location',\n", + " 'description': 'Stores names of geographical locations of different types (worldwide, country, region).'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'code': {'type': 'string',\n", + " 'required': True,\n", + " 'unique': True,\n", + " 'description': 'Unique textual identifier for the location, e.g. iso3 code for countries.'},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'totalMarineArea': {'type': 'integer',\n", + " 'description': 'Total marine area in km2',\n", + " 'required': True,\n", + " 'min': 0},\n", + " 'type': {'type': 'string', 'required': True},\n", + " 'groups': {'type': 'relation',\n", + " 'relation': 'manyToMany',\n", + " 'target': 'api::location.location',\n", + " 'mappedBy': 'members',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'manyToMany'},\n", + " 'members': {'type': 'relation',\n", + " 'relation': 'manyToMany',\n", + " 'target': 'api::location.location',\n", + " 'inversedBy': 'groups',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'manyToMany'},\n", + " 'fishing_protection_level_stats': {'type': 'relation',\n", + " 'relation': 'oneToMany',\n", + " 'target': 'api::fishing-protection-level-stat.fishing-protection-level-stat',\n", + " 'mappedBy': 'location',\n", + " 'targetModel': 'api::fishing-protection-level-stat.fishing-protection-level-stat',\n", + " 'relationType': 'oneToMany'},\n", + " 'mpaa_protection_level_stats': {'type': 'relation',\n", + " 'relation': 'oneToMany',\n", + " 'target': 'api::mpaa-protection-level-stat.mpaa-protection-level-stat',\n", + " 'mappedBy': 'location',\n", + " 'targetModel': 'api::mpaa-protection-level-stat.mpaa-protection-level-stat',\n", + " 'relationType': 'oneToMany'},\n", + " 'protection_coverage_stats': {'type': 'relation',\n", + " 'relation': 'oneToMany',\n", + " 'target': 'api::protection-coverage-stat.protection-coverage-stat',\n", + " 'mappedBy': 'location',\n", + " 'targetModel': 'api::protection-coverage-stat.protection-coverage-stat',\n", + " 'relationType': 'oneToMany'},\n", + " 'bounds': {'type': 'json'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpa.mpa',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpa',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpa',\n", + " 'pluralName': 'mpas',\n", + " 'displayName': 'MPA',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'wdpaid': {'type': 'integer', 'required': False},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'area': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'year': {'type': 'integer', 'min': 0},\n", + " 'mpaa_establishment_stage': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::mpaa-establishment-stage.mpaa-establishment-stage',\n", + " 'targetModel': 'api::mpaa-establishment-stage.mpaa-establishment-stage',\n", + " 'relationType': 'oneToOne'},\n", + " 'protection_status': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::protection-status.protection-status',\n", + " 'targetModel': 'api::protection-status.protection-status',\n", + " 'relationType': 'oneToOne'},\n", + " 'mpa_protection_coverage_stats': {'type': 'relation',\n", + " 'relation': 'oneToMany',\n", + " 'target': 'api::mpa-protection-coverage-stat.mpa-protection-coverage-stat',\n", + " 'mappedBy': 'mpa',\n", + " 'targetModel': 'api::mpa-protection-coverage-stat.mpa-protection-coverage-stat',\n", + " 'relationType': 'oneToMany'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpa-protection-coverage-stat.mpa-protection-coverage-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpa-protection-coverage-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpa-protection-coverage-stat',\n", + " 'pluralName': 'mpa-protection-coverage-stats',\n", + " 'displayName': 'MPA Protection Coverage Stats',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'mpa': {'type': 'relation',\n", + " 'relation': 'manyToOne',\n", + " 'target': 'api::mpa.mpa',\n", + " 'inversedBy': 'mpa_protection_coverage_stats',\n", + " 'targetModel': 'api::mpa.mpa',\n", + " 'relationType': 'manyToOne'},\n", + " 'fishing_protection_level': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::fishing-protection-level.fishing-protection-level',\n", + " 'targetModel': 'api::fishing-protection-level.fishing-protection-level',\n", + " 'relationType': 'oneToOne'},\n", + " 'mpaa_protection_level': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::mpaa-protection-level.mpaa-protection-level',\n", + " 'targetModel': 'api::mpaa-protection-level.mpaa-protection-level',\n", + " 'relationType': 'oneToOne'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::location.location',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'oneToOne'},\n", + " 'area': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpaa-establishment-stage.mpaa-establishment-stage',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpaa-establishment-stage',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpaa-establishment-stage',\n", + " 'pluralName': 'mpaa-establishment-stages',\n", + " 'displayName': 'MPAA Establishment Stage'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'info': {'type': 'text'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpaa-establishment-stage-stat.mpaa-establishment-stage-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpaa-establishment-stage-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpaa-establishment-stage-stat',\n", + " 'pluralName': 'mpaa-establishment-stage-stats',\n", + " 'displayName': 'MPAA Establishment Stage Stats'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::location.location',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'oneToOne'},\n", + " 'mpaa_establishment_stage': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::mpaa-establishment-stage.mpaa-establishment-stage',\n", + " 'targetModel': 'api::mpaa-establishment-stage.mpaa-establishment-stage',\n", + " 'relationType': 'oneToOne'},\n", + " 'protection_status': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::protection-status.protection-status',\n", + " 'targetModel': 'api::protection-status.protection-status',\n", + " 'relationType': 'oneToOne'},\n", + " 'year': {'type': 'integer', 'required': True, 'min': 0},\n", + " 'area': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpaa-protection-level.mpaa-protection-level',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpaa-protection-level',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpaa-protection-level',\n", + " 'pluralName': 'mpaa-protection-levels',\n", + " 'displayName': 'MPAA Protection Level'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'info': {'type': 'text'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::mpaa-protection-level-stat.mpaa-protection-level-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'mpaa-protection-level-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'mpaa-protection-level-stat',\n", + " 'pluralName': 'mpaa-protection-level-stats',\n", + " 'displayName': 'MPAA Protection Level Stats',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'manyToOne',\n", + " 'target': 'api::location.location',\n", + " 'inversedBy': 'mpaa_protection_level_stats',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'manyToOne'},\n", + " 'mpaa_protection_level': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::mpaa-protection-level.mpaa-protection-level',\n", + " 'targetModel': 'api::mpaa-protection-level.mpaa-protection-level',\n", + " 'relationType': 'oneToOne'},\n", + " 'area': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::protection-coverage-stat.protection-coverage-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'protection-coverage-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'protection-coverage-stat',\n", + " 'pluralName': 'protection-coverage-stats',\n", + " 'displayName': 'Protection Coverage Stats',\n", + " 'description': ''},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'manyToOne',\n", + " 'target': 'api::location.location',\n", + " 'inversedBy': 'protection_coverage_stats',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'manyToOne'},\n", + " 'protection_status': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::protection-status.protection-status',\n", + " 'targetModel': 'api::protection-status.protection-status',\n", + " 'relationType': 'oneToOne'},\n", + " 'year': {'type': 'integer', 'required': True, 'min': 0},\n", + " 'cumSumProtectedArea': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'protectedArea': {'type': 'decimal',\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'protectedAreasCount': {'type': 'integer', 'required': True},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}},\n", + " {'uid': 'api::protection-status.protection-status',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'protection-status',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'protection-status',\n", + " 'pluralName': 'protection-statuses',\n", + " 'displayName': 'Protection Status'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'slug': {'type': 'string', 'required': True, 'unique': True},\n", + " 'name': {'type': 'string', 'required': True},\n", + " 'info': {'type': 'text'},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}}]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "base_url = 'http://192.168.50.116:1337'\n", + "\n", + "strapi = Strapi(url=base_url)\n", + "strapi.login()\n", + "strapi.getCollections" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d58ab000", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'uid': 'api::habitat-stat.habitat-stat',\n", + " 'isDisplayed': True,\n", + " 'apiID': 'habitat-stat',\n", + " 'kind': 'collectionType',\n", + " 'info': {'singularName': 'habitat-stat',\n", + " 'pluralName': 'habitat-stats',\n", + " 'displayName': 'Habitat Stats',\n", + " 'description': 'Calculation of area of protection by location, habitat and year'},\n", + " 'options': {'draftAndPublish': False},\n", + " 'pluginOptions': {},\n", + " 'attributes': {'id': {'type': 'integer'},\n", + " 'location': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::location.location',\n", + " 'targetModel': 'api::location.location',\n", + " 'relationType': 'oneToOne'},\n", + " 'habitat': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'api::habitat.habitat',\n", + " 'targetModel': 'api::habitat.habitat',\n", + " 'relationType': 'oneToOne'},\n", + " 'year': {'type': 'integer', 'required': True},\n", + " 'protectedArea': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'totalArea': {'type': 'decimal',\n", + " 'required': True,\n", + " 'min': 0,\n", + " 'column': {'defaultTo': 0, 'type': 'decimal', 'args': [12, 2]}},\n", + " 'createdAt': {'type': 'datetime'},\n", + " 'updatedAt': {'type': 'datetime'},\n", + " 'createdBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'},\n", + " 'updatedBy': {'type': 'relation',\n", + " 'relation': 'oneToOne',\n", + " 'target': 'admin::user',\n", + " 'configurable': False,\n", + " 'writable': False,\n", + " 'visible': False,\n", + " 'useJoinTable': False,\n", + " 'private': True,\n", + " 'targetModel': 'admin::user',\n", + " 'relationType': 'oneToOne'}}}]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "strapi.getCollectionMetadata('habitat-stats')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b9fdf6a9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': [{'id': 1,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 111638.25,\n", + " 'totalArea': 224435.08,\n", + " 'createdAt': '2023-10-09T17:57:00.168Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.262Z'}},\n", + " {'id': 2,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 74787.45,\n", + " 'totalArea': 314001.94,\n", + " 'createdAt': '2023-10-09T17:57:24.349Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.272Z'}},\n", + " {'id': 3,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 63259.5,\n", + " 'totalArea': 149886.97,\n", + " 'createdAt': '2023-10-09T17:57:55.786Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.282Z'}},\n", + " {'id': 4,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 4400.14,\n", + " 'totalArea': 15336.98,\n", + " 'createdAt': '2023-10-09T17:58:29.609Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.296Z'}},\n", + " {'id': 5,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 61287.2,\n", + " 'totalArea': 147358.99,\n", + " 'createdAt': '2023-10-09T17:58:54.453Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.308Z'}},\n", + " {'id': 6,\n", + " 'attributes': {'year': 2023,\n", + " 'protectedArea': 297168.14,\n", + " 'totalArea': 352349.89,\n", + " 'createdAt': '2023-10-09T17:59:25.804Z',\n", + " 'updatedAt': '2023-11-02T10:59:27.321Z'}}],\n", + " 'meta': {'pagination': {'page': 1,\n", + " 'pageSize': 25,\n", + " 'pageCount': 1,\n", + " 'total': 6},\n", + " 'updatedAt': '2023-11-02T10:59:27.321Z'}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "strapi.getCollectionData('habitat-stats')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "445bd526", + "metadata": {}, + "outputs": [], + "source": [ + "strapi.importCollection('habitat-stat', '/home/mambauser/data/test_statistics_upload/rename_data.csv')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/data/notebooks/tiles_generation/EEZ_tiles.ipynb b/data/notebooks/tiles_generation/EEZ_tiles.ipynb deleted file mode 100644 index 09816d0b..00000000 --- a/data/notebooks/tiles_generation/EEZ_tiles.ipynb +++ /dev/null @@ -1,157 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "79966c59", - "metadata": {}, - "source": [ - "This notebook relies on the following cmd line tools, ensure they are installed and in the system\n", - "- tippecanoe\n", - "- mapshaper\n", - "- aws cli" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "def1ef65-1fd6-4367-a620-2dbe415f7104", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import sys\n", - "import logging\n", - "\n", - "from IPython.lib import backgroundjobs as bg\n", - "\n", - "scripts_dir = Path('../..').joinpath('src')\n", - "if scripts_dir not in sys.path:\n", - " sys.path.insert(0, scripts_dir.resolve().as_posix())\n", - " \n", - "from pipelines.pipes import get_pipes\n", - "from strapi import Strapi\n", - "\n", - "logging.basicConfig(level=logging.DEBUG)\n", - "logging.getLogger(\"requests\").setLevel(logging.WARNING)\n", - "logging.getLogger(\"urllib3\").setLevel(logging.WARNING)\n", - "jobs = bg.BackgroundJobManager()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "eb61ff00", - "metadata": {}, - "outputs": [ - { - "ename": "HTTPError", - "evalue": "404 Client Error: Not Found for url: http://192.168.50.116:1337/content-manager/collection-types", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/alicitita/Projects/skytruth-30x30/data/notebooks/tiles_generation/EEZ_tiles.ipynb Cell 3\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 1\u001b[0m strapi \u001b[39m=\u001b[39m Strapi(url\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mhttp://192.168.50.116:1337\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 2\u001b[0m strapi\u001b[39m.\u001b[39mlogin(\u001b[39m'\u001b[39m\u001b[39malicia\u001b[39m\u001b[39m'\u001b[39m,\u001b[39m'\u001b[39m\u001b[39mKitiara_90\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 3\u001b[0m )\n\u001b[0;32m----> 5\u001b[0m strapi\u001b[39m.\u001b[39;49mgetCollections\n", - "File \u001b[0;32m/opt/conda/lib/python3.10/functools.py:981\u001b[0m, in \u001b[0;36mcached_property.__get__\u001b[0;34m(self, instance, owner)\u001b[0m\n\u001b[1;32m 979\u001b[0m val \u001b[39m=\u001b[39m cache\u001b[39m.\u001b[39mget(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mattrname, _NOT_FOUND)\n\u001b[1;32m 980\u001b[0m \u001b[39mif\u001b[39;00m val \u001b[39mis\u001b[39;00m _NOT_FOUND:\n\u001b[0;32m--> 981\u001b[0m val \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfunc(instance)\n\u001b[1;32m 982\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 983\u001b[0m cache[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mattrname] \u001b[39m=\u001b[39m val\n", - "File \u001b[0;32m~/src/strapi.py:53\u001b[0m, in \u001b[0;36mStrapi.getCollections\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[39m@cached_property\u001b[39m\n\u001b[1;32m 51\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgetCollections\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 52\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msession\u001b[39m.\u001b[39mget(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m/content-manager/collection-types\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m---> 53\u001b[0m response\u001b[39m.\u001b[39;49mraise_for_status()\n\u001b[1;32m 54\u001b[0m data \u001b[39m=\u001b[39m [\n\u001b[1;32m 55\u001b[0m response\u001b[39m.\u001b[39mjson()\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 56\u001b[0m \u001b[39mfor\u001b[39;00m collection \u001b[39min\u001b[39;00m response\u001b[39m.\u001b[39mjson()\n\u001b[1;32m 57\u001b[0m \u001b[39mif\u001b[39;00m collection\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39misDisplayed\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m==\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ]\n\u001b[1;32m 59\u001b[0m \u001b[39mreturn\u001b[39;00m data\n", - "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/requests/models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1016\u001b[0m http_error_msg \u001b[39m=\u001b[39m (\n\u001b[1;32m 1017\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstatus_code\u001b[39m}\u001b[39;00m\u001b[39m Server Error: \u001b[39m\u001b[39m{\u001b[39;00mreason\u001b[39m}\u001b[39;00m\u001b[39m for url: \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1018\u001b[0m )\n\u001b[1;32m 1020\u001b[0m \u001b[39mif\u001b[39;00m http_error_msg:\n\u001b[0;32m-> 1021\u001b[0m \u001b[39mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m)\n", - "\u001b[0;31mHTTPError\u001b[0m: 404 Client Error: Not Found for url: http://192.168.50.116:1337/content-manager/collection-types" - ] - } - ], - "source": [ - "strapi = Strapi(url='http://192.168.50.116:1337')\n", - "strapi.login()\n", - "\n", - "strapi.getCollections" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c4b3fc7", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:pipelines.settings:/home/mambauser/data\n", - "DEBUG:root:Starting job # 0 in a separate thread.\n", - "INFO:pipelines.base_pipe:Running eez_tiles pipeline\n", - "INFO:utils:File /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118.zip already exists.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:utils:Unzip Finish.\n", - "Allocating 8 GB of heap memory\n", - "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.shp\n", - "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.shx\n", - "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.dbf\n", - "[o] Wrote /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.prj\n", - "INFO:mapbox_uploader:Uploading to Mapbox...\n", - "INFO:mapbox_uploader:Uploading to S3...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 69.8 MiB/69.8 MiB (7.8 MiB/s) with 1 file(s) remaining \r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:mapbox_uploader:CompletedProcess(args='aws s3 cp /home/mambauser/data/eez_tiles/World_EEZ_v11_20191118/eez_v11.mbtiles s3://tilestream-tilesets-production/53/_pending/h75tk8ezapo2qst37s0l04olc/skytruth --region us-east-1', returncode=0)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "upload: ../../data/eez_tiles/World_EEZ_v11_20191118/eez_v11.mbtiles to s3://tilestream-tilesets-production/53/_pending/h75tk8ezapo2qst37s0l04olc/skytruth\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 100/100 [01:48<00:00, 1.08s/it]\n", - "INFO:mapbox_uploader:True\n" - ] - } - ], - "source": [ - "# this code will execute all pipelines selected in background.\n", - "mypipes_subset = {k: v for k, v in get_pipes().items() if k in ['eez_tiles']}\n", - "for name, pipe in mypipes_subset.items():\n", - " jobs.new(pipe().execute)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/data/src/strapi.py b/data/src/strapi.py index e369bdb2..92720b18 100644 --- a/data/src/strapi.py +++ b/data/src/strapi.py @@ -57,24 +57,32 @@ def logout(self) -> None: @cached_property def getCollections(self): - response = self.session.get(f"{self.url}/content-manager/collection-types") + response = self.session.get(f"{self.url}/content-manager/content-types") response.raise_for_status() - data = [ - response.json().get("data") - for collection in response.json() - if collection.get("isDisplayed") == True - ] + data = list( + filter( + lambda x: x.get("isDisplayed") == True and "api::" in x.get("uid"), + response.json().get("data"), + ) + ) return data - def getCollectionMetadata(self, collectionName): - return list(filter(lambda x: (collectionName in x), self.getCollections)) + def getCollectionMetadata(self, name): + return list( + filter( + lambda x: (name in x.get("info", {}).values()), + self.getCollections, + ) + ) - def getCollectionData(self, collectionName): - response = self.session.get(f"{self.url}/api/{collectionName}") + def getCollectionData(self, collectionUid: str): + response = self.session.get(f"{self.url}/api/{collectionUid}") response.raise_for_status() return response.json() - def importCollection(self, collectionName: str, file_path: Path, idField: str): + def importCollection( + self, collectionApiID: str, file_path: Path, idField: str = "id" + ): extension = file_path.suffix with open(file_path, "rb") as f: @@ -84,7 +92,7 @@ def importCollection(self, collectionName: str, file_path: Path, idField: str): f"{self.url}/api/import-export-entries/content/import", json={ "idField": idField, - "slug": f"api::{collectionName}.{collectionName}", + "slug": f"api::{collectionApiID}.{collectionApiID}", "data": data, "format": extension, },