diff --git a/cloud_functions/analysis/docker-compose.yml b/cloud_functions/analysis/docker-compose.yml index 7561beef..1a76e7ab 100644 --- a/cloud_functions/analysis/docker-compose.yml +++ b/cloud_functions/analysis/docker-compose.yml @@ -25,7 +25,7 @@ services: skytruth-db-init: profiles: - feed_data_to_db - image: ghcr.io/osgeo/gdal:ubuntu-small-latest + image: ghcr.io/osgeo/gdal:ubuntu-small-3.9.3 container_name: skytruth_cf_db_init environment: GRANT_SUDO: "yes" @@ -33,7 +33,7 @@ services: restart: "no" volumes: - ./init.sh:/usr/share/init.sh - entrypoint: bash -c "chmod +x /usr/share/init.sh && /usr/share/init.sh" + entrypoint: bash -c "chmod +x /usr/share/init.sh && /usr/share/init.sh eez gadm" network_mode: "host" extra_hosts: - docker.host:0.0.0.0 diff --git a/cloud_functions/analysis/init.sh b/cloud_functions/analysis/init.sh index 0eae16d4..e294431c 100755 --- a/cloud_functions/analysis/init.sh +++ b/cloud_functions/analysis/init.sh @@ -1,16 +1,47 @@ #!/bin/sh - -URL='vector-data-raw/vizzuality_processed_data/analysis_data/eez_minus_mpa.zip' +# Upload the data to the database +# Usage: init.sh [OPTION] +for arg in "$@" +do + case $arg in + eez) + echo "uploading eez_minus_mpa" + URL='vector-data-raw/vizzuality_processed_data/analysis_data/eez_minus_mpa.zip' + TABLE_NAME='eez_minus_mpa' + ;; + gadm) + echo "uploading gadm_minus_pa" + URL='vector-data-raw/vizzuality_processed_data/analysis_data/gadm_minus_pa.zip' + TABLE_NAME='gadm_minus_pa' + ;; + --help) + echo "Usage: init.sh [OPTION]" + echo "Upload the data to the database" + echo "" + echo "Options:" + echo " --eez Upload eez_minus_mpa data" + echo " --gadm Upload gadm_minus_pa data" + echo " --help Display this help message" + exit 0 + ;; + *) + echo "Invalid option $arg" + exit 1 + ;; + esac +ogr2ogr --version ogr2ogr -progress \ - -makevalid -overwrite \ - -nln eez_minus_mpa -nlt PROMOTE_TO_MULTI \ - -lco GEOMETRY_NAME=the_geom \ - -lco PRECISION=FALSE \ - -lco SPATIAL_INDEX=GIST \ - -lco FID=id \ - -t_srs EPSG:4326 -a_srs EPSG:4326 \ - -f PostgreSQL PG:"host=$POSTGRES_HOST port=$POSTGRES_PORT \ - user=$POSTGRES_USER password=$POSTGRES_PASSWORD \ - dbname=$POSTGRES_DB active_schema=$POSTGRES_SCHEMA" \ - -doo "PRELUDE_STATEMENTS=CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION CURRENT_USER;" "/vsizip/vsigs/$URL"; \ No newline at end of file + -makevalid -overwrite \ + -nln $TABLE_NAME -nlt PROMOTE_TO_MULTI \ + -lco GEOMETRY_NAME=the_geom \ + -lco PRECISION=FALSE \ + -lco SPATIAL_INDEX=GIST \ + -lco FID=id \ + -t_srs EPSG:4326 \ + -f PostgreSQL PG:"host=$POSTGRES_HOST port=$POSTGRES_PORT \ + user=$POSTGRES_USER password=$POSTGRES_PASSWORD \ + dbname=$POSTGRES_DB active_schema=$POSTGRES_SCHEMA" \ + -doo "PRELUDE_STATEMENTS=CREATE SCHEMA IF NOT EXISTS $POSTGRES_SCHEMA AUTHORIZATION CURRENT_USER;" "/vsizip/vsigs/$URL"; + +done \ No newline at end of file diff --git a/cloud_functions/analysis/main.py b/cloud_functions/analysis/main.py index f2b47d62..d8e7662d 100644 --- a/cloud_functions/analysis/main.py +++ b/cloud_functions/analysis/main.py @@ -3,7 +3,7 @@ import logging from src.connect_tcp import connect_tcp_socket -from src.analysis import get_locations_stats +from src.analysis import get_locations_stats_terrestrial, get_locations_stats_marine logger = logging.getLogger(__name__) logger.addHandler(default_handler) @@ -50,7 +50,19 @@ def index(request): if not geometry: raise ValueError("geometry is required") - return (get_locations_stats(db, geometry), 200, headers) + funct = { + "marine": get_locations_stats_marine, + "terrestrial": get_locations_stats_terrestrial, + } + + environment = ({**request.args, **request.get_json()}).get( + "environment", "marine" + ) + if environment not in funct.keys(): + raise ValueError("environment must be one of `marine` or `terrestrial`") + + return (funct[environment](db, geometry), 200, headers) + except ValueError as e: logger.exception(str(e)) return {"error": str(e)}, 400, headers diff --git a/cloud_functions/analysis/src/analysis.py b/cloud_functions/analysis/src/analysis.py index 583c4bc2..cdc37dd2 100644 --- a/cloud_functions/analysis/src/analysis.py +++ b/cloud_functions/analysis/src/analysis.py @@ -12,8 +12,10 @@ def get_geojson(geojson: JSON) -> dict: else: return geojson +### Marine -def serialize_response(data: dict) -> dict: + +def serialize_response_marine(data: dict) -> dict: """Converts the data from the database into a Dict {locations_area:{"code":, "protected_area": , "area":}, "total_area":} response """ @@ -48,7 +50,9 @@ def serialize_response(data: dict) -> dict: return result -def get_locations_stats(db: sqlalchemy.engine.base.Engine, geojson: JSON) -> dict: +def get_locations_stats_marine( + db: sqlalchemy.engine.base.Engine, geojson: JSON +) -> dict: with db.connect() as conn: stmt = sqlalchemy.text( """ @@ -65,4 +69,62 @@ def get_locations_stats(db: sqlalchemy.engine.base.Engine, geojson: JSON) -> dic stmt, parameters={"geometry": get_geojson(geojson)} ).all() - return serialize_response(data_response) + return serialize_response_marine(data_response) + + +### Terrestrial +def serialize_response_terrestrial(data: dict) -> dict: + """Converts the data from the database + into a Dict {locations_area:{"code":, "protected_area": , "area":}, "total_area":} response + """ + if not data or len(data) == 0: + raise ValueError( + "No data found, this is likely due to a geometry that does not intersect with a Marine area." + ) + + result = {"total_area": data[0][3]} + sub_result = {} + total_protected_area = 0 + for row in data: + for iso in filter(lambda item: item is not None, [row[1]]): + total_protected_area += row[2] + if iso not in sub_result: + sub_result[iso] = { + "code": iso, + "protected_area": row[2], + "area": row[0], + } + else: + sub_result[iso]["protected_area"] += row[2] + sub_result[iso]["area"] += row[0] + + result.update( + { + "locations_area": list(sub_result.values()), + "total_protected_area": total_protected_area, + } + ) + + return result + + +def get_locations_stats_terrestrial( + db: sqlalchemy.engine.base.Engine, geojson: JSON +) -> dict: + with db.connect() as conn: + stmt = sqlalchemy.text( + """ + with user_data as (select ST_GeomFromGeoJSON(:geometry) as geom), + user_data_stats as (select *, round((st_area(st_transform(geom,'+proj=longlat +datum=WGS84 +no_defs +type=crs', '+proj=moll +lon_0=0 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs +type=crs'))/1e6)) user_area_km2 from user_data), + stats as (select area_km2,gid_0, round((st_area(st_transform(st_makevalid(st_intersection(the_geom, user_data_stats.geom)),'+proj=longlat +datum=WGS84 +no_defs +type=crs', '+proj=moll +lon_0=0 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs +type=crs'))/1e6)) portion_area_km2, + user_data_stats.user_area_km2 + from data.gadm_minus_pa gmp , user_data_stats + where st_intersects(the_geom, user_data_stats.geom)) + select avg(area_km2) as area_km2, gid_0, sum(portion_area_km2) as portion_area_km2, avg(user_area_km2) as user_area_km2 from stats group by gid_0 + """ + ) + data_response = conn.execute( + stmt, parameters={"geometry": get_geojson(geojson)} + ).all() + + return serialize_response_terrestrial(data_response)