From 4dd430414612cba45fe02e18aebd87487f8e9f8f Mon Sep 17 00:00:00 2001 From: Sujan Adhikari Date: Fri, 8 Nov 2024 17:02:44 +0545 Subject: [PATCH] fix(additional-entity): allow custom properties to create entities list --- src/backend/app/central/central_crud.py | 20 ++++++------- src/backend/app/central/central_schemas.py | 34 ++++++++++++++++++++-- src/backend/app/projects/project_crud.py | 8 +++++ src/backend/app/projects/project_routes.py | 5 +++- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 6bf8916339..7d28631457 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -506,12 +506,14 @@ async def feature_geojson_to_entity_dict( raise ValueError(msg) javarosa_geom = await geojson_to_javarosa_geom(geometry) + properties = {} + for key, value in feature.get("properties", {}).items(): + if not central_schemas.is_valid_property_name(key): + log.warning(f"Invalid property name: {key},Excluding from properties.") + continue + # NOTE all properties MUST be string values for Entities, convert + properties.update({str(central_schemas.sanitize_key(key)): str(value)}) - # NOTE all properties MUST be string values for Entities, convert - properties = { - str(key): str(value) for key, value in feature.get("properties", {}).items() - } - # Set to MappingState enum READY value (0) properties["status"] = "0" task_id = properties.get("task_id") @@ -540,15 +542,13 @@ async def task_geojson_dict_to_entity_values( async def create_entity_list( odk_creds: central_schemas.ODKCentralDecrypted, odk_id: int, + properties: list[str], dataset_name: str = "features", - properties: list[str] = None, entities_list: list[central_schemas.EntityDict] = None, ) -> None: """Create a new Entity list in ODK.""" - if properties is None: - # Get the default properties for FMTM project - properties = central_schemas.entity_fields_to_list() - log.debug(f"Using default FMTM properties for Entity creation: {properties}") + log.info("Creating ODK Entity properties list") + properties = central_schemas.entity_fields_to_list(properties) async with central_deps.get_odk_dataset(odk_creds) as odk_central: # Step 1: create the Entity list, with properties diff --git a/src/backend/app/central/central_schemas.py b/src/backend/app/central/central_schemas.py index 04bce7af2c..83b739ac36 100644 --- a/src/backend/app/central/central_schemas.py +++ b/src/backend/app/central/central_schemas.py @@ -17,6 +17,7 @@ # """Schemas for returned ODK Central objects.""" +import re from dataclasses import dataclass from typing import Optional, Self, TypedDict @@ -130,10 +131,39 @@ class NameTypeMapping: NameTypeMapping(name="status", type="string"), ] +RESERVED_KEYS = { + "name", + "label", + "uuid", +} # Add any other reserved keys of odk central in here as needed +ALLOWED_PROPERTY_PATTERN = re.compile(r"^[a-zA-Z0-9_]+$") -def entity_fields_to_list() -> list[str]: + +def is_valid_property_name(name: str) -> bool: + """Check if a property name is valid according to allowed characters pattern.""" + return bool(ALLOWED_PROPERTY_PATTERN.match(name)) + + +def sanitize_key(key: str) -> str: + """Rename reserved keys to avoid conflicts with ODK Central's schema.""" + if key in RESERVED_KEYS: + return f"custom_{key}" + return key + + +def entity_fields_to_list(properties: list[str]) -> list[str]: """Converts a list of Field objects to a list of field names.""" - return [field.name for field in ENTITY_FIELDS] + sanitized_properties = [] + for property in properties: + if not is_valid_property_name(property): + log.warning(f"Invalid property name: {property},Excluding from properties.") + continue + sanitized_properties.append(sanitize_key(property)) + default_properties = [field.name for field in ENTITY_FIELDS] + for item in default_properties: + if item not in sanitized_properties: + sanitized_properties.append(item) + return sanitized_properties # Dynamically generate EntityPropertyDict using ENTITY_FIELDS diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index c334253196..3938d3fc4e 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -474,6 +474,7 @@ async def generate_odk_central_project_content( odk_credentials: central_schemas.ODKCentralDecrypted, xlsform: BytesIO, task_extract_dict: dict[int, geojson.FeatureCollection], + entity_properties: list[str], ) -> str: """Populate the project in ODK Central with XForm, Appuser, Permissions.""" # The ODK Dataset (Entity List) must exist prior to main XLSForm @@ -485,6 +486,7 @@ async def generate_odk_central_project_content( await central_crud.create_entity_list( odk_credentials, project_odk_id, + properties=entity_properties, dataset_name="features", entities_list=entities_list, ) @@ -530,6 +532,11 @@ async def generate_project_files( log.debug("Getting data extract geojson from flatgeobuf") feature_collection = await get_project_features_geojson(db, project) + # Get properties to create datasets + entity_properties = list( + feature_collection.get("features")[0].get("properties").keys() + ) + # Split extract by task area log.debug("Splitting data extract per task area") # TODO in future this splitting could be removed if the task_id is @@ -550,6 +557,7 @@ async def generate_project_files( project_odk_creds, BytesIO(project_xlsform), task_extract_dict, + entity_properties, ) log.debug( f"Setting encrypted odk token for FMTM project ({project_id}) " diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index fa69c416d1..eb2dd40f88 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -652,15 +652,18 @@ async def add_additional_entity_list( # Parse geojson + divide by task # (not technically required, but also appends properties in correct format) featcol = parse_geojson_file_to_featcol(await geojson.read()) + properties = list(featcol.get("features")[0].get("properties").keys()) feature_split_by_task = await split_geojson_by_task_areas(db, featcol, project_id) entities_list = await central_crud.task_geojson_dict_to_entity_values( feature_split_by_task ) + dataset_name = entity_name.replace(" ", "_") await central_crud.create_entity_list( project_odk_creds, project_odk_id, - dataset_name=entity_name, + properties=properties, + dataset_name=dataset_name, entities_list=entities_list, )