From 74e3d2a1c4bbaec3f8a9b12725248f1e5c076cfb Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Fri, 20 Sep 2024 22:28:44 +0100 Subject: [PATCH] feat(backend): use XLSForm injection during project creation (#1792) * refactor: remove redundant convert-xlsform-to-xform helper endpoint * refactor(frontend): renamed 'existing' xlsform field --> 'feature' * refactor(backend): replace xml manipulation with pandas update_xlsform func * build: update osm-fieldwork --> v0.16.2 * fix(backend): do not add mandatory fields to xlsform twice * fix(backend): form updating does not append mandatory fields twice * build: update osm-fieldwork --> v0.16.4 --- src/backend/app/central/central_crud.py | 272 +++++------------- src/backend/app/central/central_deps.py | 31 ++ src/backend/app/helpers/helper_routes.py | 26 -- src/backend/app/projects/project_crud.py | 47 +-- src/backend/app/projects/project_routes.py | 143 +++++---- src/backend/pdm.lock | 8 +- src/backend/pyproject.toml | 2 +- src/frontend/src/api/CreateProjectService.ts | 4 +- .../ManageProject/EditTab/FormUpdateTab.tsx | 76 ++--- .../FeatureSelectionPopup.tsx | 2 +- 10 files changed, 213 insertions(+), 398 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 7b7c3f6116..cfe1728a23 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -19,17 +19,15 @@ import csv import json -import os -import uuid from io import BytesIO, StringIO from typing import Optional, Union -from xml.etree.ElementTree import Element, SubElement import geojson from defusedxml import ElementTree from fastapi import HTTPException from loguru import logger as log from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject +from osm_fieldwork.update_xlsform import append_mandatory_fields from pyxform.xls2xform import convert as xform_convert from sqlalchemy import text from sqlalchemy.orm import Session @@ -314,11 +312,70 @@ async def get_form_list(db: Session) -> list: ) from e +async def read_and_test_xform(input_data: BytesIO) -> None: + """Read and validate an XForm. + + Args: + input_data (BytesIO): form to be tested. + + Returns: + BytesIO: the converted XML representation of the XForm. + """ + try: + log.debug("Parsing XLSForm --> XML data") + # NOTE pyxform.xls2xform.convert returns a ConvertResult object + return BytesIO(xform_convert(input_data).xform.encode("utf-8")) + except Exception as e: + log.error(e) + msg = f"XLSForm is invalid: {str(e)}" + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg + ) from e + + +async def append_fields_to_user_xlsform( + xlsform: BytesIO, + form_category: str = "buildings", + additional_entities: list[str] = None, + task_count: int = None, + existing_id: str = None, +) -> BytesIO: + """Helper to return the intermediate XLSForm prior to convert.""" + log.debug("Appending mandatory FMTM fields to XLSForm") + return await append_mandatory_fields( + xlsform, + form_category=form_category, + additional_entities=additional_entities, + task_count=task_count, + existing_id=existing_id, + ) + + +async def validate_and_update_user_xlsform( + xlsform: BytesIO, + form_category: str = "buildings", + additional_entities: list[str] = None, + task_count: int = None, + existing_id: str = None, +) -> BytesIO: + """Wrapper to append mandatory fields and validate user uploaded XLSForm.""" + updated_file_bytes = await append_fields_to_user_xlsform( + xlsform, + form_category=form_category, + additional_entities=additional_entities, + task_count=task_count, + existing_id=existing_id, + ) + + # Validate and return the form + log.debug("Validating uploaded XLS form") + return await read_and_test_xform(updated_file_bytes) + + async def update_project_xform( xform_id: str, odk_id: int, - xform_data: BytesIO, - form_file_ext: str, + xlsform: BytesIO, category: str, task_count: int, odk_credentials: project_schemas.ODKCentralDecrypted, @@ -328,220 +385,29 @@ async def update_project_xform( Args: xform_id (str): The UUID of the existing XForm in ODK Central. odk_id (int): ODK Central form ID. - xform_data (BytesIO): XForm data. - form_file_ext (str): Extension of the form file. + xlsform (UploadFile): XForm data. category (str): Category of the XForm. task_count (int): The number of tasks in a project. odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds. + + Returns: None """ - xform_data = await read_and_test_xform( - xform_data, - form_file_ext, - return_form_data=True, - ) - updated_xform_data = await modify_xform_xml( - xform_data, - category, - task_count, - existing_id=xform_id, - ) + xform_bytesio = await read_and_test_xform(xlsform) xform_obj = get_odk_form(odk_credentials) # NOTE calling createForm for an existing form will update it xform_obj.createForm( odk_id, - updated_xform_data, + xform_bytesio, form_name=xform_id, ) # The draft form must be published after upload + # NOTE we can't directly publish existing forms + # in createForm and need 2 steps xform_obj.publishForm(odk_id, xform_id) -async def read_and_test_xform( - input_data: BytesIO, - form_file_ext: str, - return_form_data: bool = False, -) -> BytesIO | dict: - """Read and validate an XForm. - - Args: - input_data (BytesIO): form to be tested. - form_file_ext (str): type of form (.xls, .xlsx, or .xml). - return_form_data (bool): return the XForm data. - """ - # Read from BytesIO object - file_ext = form_file_ext.lower() - - if file_ext == ".xml": - xform_bytesio = input_data - # Parse / validate XForm - try: - ElementTree.fromstring(xform_bytesio.getvalue()) - except ElementTree.ParseError as e: - log.error(e) - msg = f"Error parsing XForm XML: Possible reason: {str(e)}" - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg - ) from e - else: - try: - log.debug("Parsing XLSForm --> XML data") - xform_bytesio = BytesIO(xform_convert(input_data).xform.encode("utf-8")) - except Exception as e: - log.error(e) - msg = f"XLSForm is invalid: {str(e)}" - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg - ) from e - - # Return immediately - if return_form_data: - return xform_bytesio - - # Load XML - xform_xml = ElementTree.fromstring(xform_bytesio.getvalue()) - - # Extract csv filenames - try: - namespaces = {"xforms": "http://www.w3.org/2002/xforms"} - csv_list = [ - os.path.splitext(inst.attrib["src"].split("/")[-1])[0] - for inst in xform_xml.findall(".//xforms:instance[@src]", namespaces) - if inst.attrib.get("src", "").endswith(".csv") - ] - - # No select_one_from_file defined - if not csv_list: - msg = ( - "The form has no select_one_from_file or " - "select_multiple_from_file field defined for a CSV." - ) - raise ValueError(msg) from None - - return {"required_media": csv_list, "message": "Your form is valid"} - - except Exception as e: - log.error(e) - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=str(e) - ) from e - - -async def modify_xform_xml( - form_data: BytesIO, - category: str, - task_count: int, - existing_id: Optional[str] = None, -) -> BytesIO: - """Update fields in the XForm to work with FMTM. - - The 'id' field is set to random UUID (xFormId) unless existing_id is specified - The 'name' field is set to the category name. - The upload media must be equal to 'features.csv'. - The task_filter options are populated as choices in the form. - The form_category value is also injected to display in the instructions. - - Args: - form_data (str): The input form data. - category (str): The form category, used to name the dataset (entity list) - and the .csv file containing the geometries. - task_count (int): The number of tasks in a project. - existing_id (str): An existing XForm ID in ODK Central, for updating. - - Returns: - BytesIO: The XForm data. - """ - log.debug(f"Updating XML keys in survey XForm: {category}") - - if existing_id: - xform_id = existing_id - else: - xform_id = uuid.uuid4() - - namespaces = { - "h": "http://www.w3.org/1999/xhtml", - "odk": "http://www.opendatakit.org/xforms", - "xforms": "http://www.w3.org/2002/xforms", - "entities": "http://www.opendatakit.org/xforms/entities", - } - - # Parse the XML from BytesIO obj - root = ElementTree.fromstring(form_data.getvalue()) - - xform_data = root.findall(".//xforms:data[@id]", namespaces) - for dt in xform_data: - # This sets the xFormId in ODK Central (the form reference via API) - dt.set("id", str(xform_id)) - - # Update the form title (displayed in ODK Collect) - existing_title = root.find(".//h:title", namespaces) - if existing_title is not None: - existing_title.text = category - - # Update the attachment name to {category}.csv, to link to the entity list - xform_instance_src = root.findall(".//xforms:instance[@src]", namespaces) - for inst in xform_instance_src: - src_value = inst.get("src", "") - if src_value.endswith(".geojson") or src_value.endswith(".csv"): - # NOTE geojson files require jr://file/features.geojson - # NOTE csv files require jr://file-csv/features.csv - inst.set("src", "jr://file-csv/features.csv") - - # NOTE add the task ID choices to the XML - # must be defined inside root element - model_element = root.find(".//xforms:model", namespaces) - # The existing dummy value for task_filter must be removed - existing_instance = model_element.find( - ".//xforms:instance[@id='task_filter']", namespaces - ) - if existing_instance is not None: - model_element.remove(existing_instance) - # Create a new instance element - instance_task_filters = Element("instance", id="task_filter") - root_element = SubElement(instance_task_filters, "root") - # Create sub-elements for each task ID, pairs - for task_id in range(1, task_count + 1): - item = SubElement(root_element, "item") - SubElement(item, "itextId").text = f"task_filter-{task_id}" - SubElement(item, "name").text = str(task_id) - model_element.append(instance_task_filters) - - # Add task_filter choice translations (necessary to be visible in form) - itext_element = root.find(".//xforms:itext", namespaces) - if itext_element is not None: - existing_translations = itext_element.findall( - ".//xforms:translation", namespaces - ) - for translation in existing_translations: - # Remove dummy value from existing translations - existing_text = translation.find( - ".//xforms:text[@id='task_filter-0']", namespaces - ) - if existing_text is not None: - translation.remove(existing_text) - - # Append new elements for each task_id - for task_id in range(1, task_count + 1): - new_text = Element("text", id=f"task_filter-{task_id}") - value_element = Element("value") - value_element.text = str(task_id) - new_text.append(value_element) - translation.append(new_text) - - # Hardcode the form_category value for the start instructions - form_category_update = root.find( - ".//xforms:bind[@nodeset='/data/form_category']", namespaces - ) - if form_category_update is not None: - if category.endswith("s"): - # Plural to singular - category = category[:-1] - form_category_update.set("calculate", f"once('{category.rstrip('s')}')") - - return BytesIO(ElementTree.tostring(root)) - - async def convert_geojson_to_odk_csv( input_geojson: BytesIO, ) -> StringIO: diff --git a/src/backend/app/central/central_deps.py b/src/backend/app/central/central_deps.py index f1b2905e0e..4951d9813a 100644 --- a/src/backend/app/central/central_deps.py +++ b/src/backend/app/central/central_deps.py @@ -19,7 +19,11 @@ """ODK Central dependency wrappers.""" from contextlib import asynccontextmanager +from io import BytesIO +from pathlib import Path +from typing import Optional +from fastapi import File, UploadFile from fastapi.exceptions import HTTPException from osm_fieldwork.OdkCentralAsync import OdkDataset @@ -41,3 +45,30 @@ async def get_odk_dataset(odk_creds: ODKCentralDecrypted): raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, detail=str(conn_error) ) from conn_error + + +async def validate_xlsform_extension(xlsform: UploadFile): + """Validate an XLSForm has .xls or .xlsx extension.""" + file = Path(xlsform.filename) + file_ext = file.suffix.lower() + + allowed_extensions = [".xls", ".xlsx"] + if file_ext not in allowed_extensions: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="Provide a valid .xls or .xlsx file", + ) + return BytesIO(await xlsform.read()) + + +async def read_xlsform(xlsform: UploadFile) -> BytesIO: + """Read an XLSForm, validate extension, return wrapped in BytesIO.""" + return await validate_xlsform_extension(xlsform) + + +async def read_optional_xlsform( + xlsform: Optional[UploadFile] = File(None), +) -> Optional[BytesIO]: + """Read an XLSForm, validate extension, return wrapped in BytesIO.""" + if xlsform: + return await validate_xlsform_extension(xlsform) diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index 43ec2d4a31..76b397db0d 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -42,7 +42,6 @@ from app.central.central_crud import ( convert_geojson_to_odk_csv, convert_odk_submission_json_to_geojson, - read_and_test_xform, ) from app.config import settings from app.db.postgis_utils import ( @@ -114,31 +113,6 @@ async def append_required_geojson_properties( ) -@router.post("/convert-xlsform-to-xform") -async def convert_xlsform_to_xform( - xlsform: UploadFile, - current_user: AuthUser = Depends(login_required), -): - """Convert XLSForm to XForm XML.""" - filename = Path(xlsform.filename) - file_ext = filename.suffix.lower() - - allowed_extensions = [".xls", ".xlsx"] - if file_ext not in allowed_extensions: - raise HTTPException( - status_code=HTTPStatus.BAD_REQUEST, - detail="Provide a valid .xls or .xlsx file", - ) - - contents = await xlsform.read() - xform_data = await read_and_test_xform( - BytesIO(contents), file_ext, return_form_data=True - ) - - headers = {"Content-Disposition": f"attachment; filename={filename.stem}.xml"} - return Response(xform_data.getvalue(), headers=headers) - - @router.post("/convert-geojson-to-odk-csv") async def convert_geojson_to_odk_csv_wrapper( geojson: UploadFile, diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index c9c0f7eff1..89665ef8ee 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -35,7 +35,6 @@ from geojson.feature import Feature, FeatureCollection from loguru import logger as log from osm_fieldwork.basemapper import create_basemap_file -from osm_fieldwork.xlsforms import xlsforms_path from osm_rawdata.postgres import PostgresClient from shapely.geometry import shape from sqlalchemy import and_, column, func, select, table, text @@ -838,8 +837,6 @@ async def generate_odk_central_project_content( project: db_models.DbProject, odk_credentials: project_schemas.ODKCentralDecrypted, xlsform: BytesIO, - form_category: str, - form_file_ext: str, task_extract_dict: dict, db: Session, ) -> str: @@ -860,20 +857,18 @@ async def generate_odk_central_project_content( entities_list, ) - xform = await central_crud.read_and_test_xform( - xlsform, form_file_ext, return_form_data=True - ) - # Manually modify fields in XML specific to project (id, name, etc) - updated_xform = await central_crud.modify_xform_xml( - xform, - form_category, - len(task_extract_dict.keys()), - ) + # TODO add here additional upload of Entities + # TODO add code here + # additional_entities = ["roads"] + + # Do final check of XLSForm validity + return parsed XForm + xform = await central_crud.read_and_test_xform(xlsform) + # Upload survey XForm log.info("Uploading survey XForm to ODK Central") xform_id = central_crud.create_odk_xform( project_odk_id, - updated_xform, + xform, odk_credentials, ) @@ -889,7 +884,11 @@ async def generate_odk_central_project_content( ) db.execute( sql, - {"project_id": project.id, "xform_id": xform_id, "category": form_category}, + { + "project_id": project.id, + "xform_id": xform_id, + "category": project.xform_category, + }, ) db.commit() return await central_crud.get_appuser_token( @@ -900,8 +899,6 @@ async def generate_odk_central_project_content( async def generate_project_files( db: Session, project_id: int, - custom_form: Optional[BytesIO], - form_file_ext: str, background_task_id: Optional[uuid.UUID] = None, ) -> None: """Generate the files for a project. @@ -911,27 +908,13 @@ async def generate_project_files( Args: db (Session): the database session. project_id(int): id of the FMTM project. - custom_form (BytesIO): the xls file to upload if we have a custom form - form_file_ext (str): weather the form is xls, xlsx or xml background_task_id (uuid): the task_id of the background task. """ try: project = await project_deps.get_project_by_id(db, project_id) - form_category = project.xform_category log.info(f"Starting generate_project_files for project {project_id}") odk_credentials = await project_deps.get_odk_credentials(db, project_id) - if custom_form: - log.debug("User provided custom XLSForm") - xlsform = custom_form - else: - log.debug(f"Using default XLSForm for category: '{form_category}'") - - form_filename = XLSFormType(form_category).name - xlsform_path = f"{xlsforms_path}/{form_filename}.xls" - with open(xlsform_path, "rb") as f: - xlsform = BytesIO(f.read()) - # Extract data extract from flatgeobuf log.debug("Getting data extract geojson from flatgeobuf") feature_collection = await get_project_features_geojson(db, project) @@ -955,9 +938,7 @@ async def generate_project_files( encrypted_odk_token = await generate_odk_central_project_content( project, odk_credentials, - xlsform, - form_category, - form_file_ext, + BytesIO(project.form_xls), task_extract_dict, db, ) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index ac395022ed..eae5363b2d 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -40,7 +40,6 @@ from loguru import logger as log from osm_fieldwork.data_models import data_models_path from osm_fieldwork.make_data_extract import getChoices -from osm_fieldwork.update_form import update_xls_form from osm_fieldwork.xlsforms import xlsforms_path from sqlalchemy.orm import Session from sqlalchemy.sql import text @@ -48,7 +47,7 @@ from app.auth.auth_schemas import AuthUser, OrgUserDict, ProjectUserDict from app.auth.osm import login_required from app.auth.roles import mapper, org_admin, project_manager -from app.central import central_crud, central_schemas +from app.central import central_crud, central_deps, central_schemas from app.db import database, db_models from app.db.postgis_utils import ( check_crs, @@ -661,41 +660,39 @@ async def task_split( @router.post("/validate-form") -async def validate_form(form: UploadFile): - """Tests the validity of the xls form uploaded. +async def validate_form( + xlsform: BytesIO = Depends(central_deps.read_xlsform), + debug: bool = False, +): + """Basic validity check for uploaded XLSForm. - Parameters: - - form: The xls form to validate + Does not append all addition values to make this a valid FMTM form for mapping. """ - file = Path(form.filename) - file_ext = file.suffix.lower() - - allowed_extensions = [".xls", ".xlsx", ".xml"] - if file_ext not in allowed_extensions: - raise HTTPException( - status_code=400, detail="Provide a valid .xls,.xlsx,.xml file" + if debug: + updated_form = await central_crud.append_fields_to_user_xlsform( + xlsform, + task_count=1, # NOTE this must be included to append task_filter choices ) - - contents = await form.read() - updated_file_bytes = update_xls_form(BytesIO(contents)) - - # open bytes again to avoid I/O error on closed bytes - form_data = BytesIO(updated_file_bytes.getvalue()) - - await central_crud.read_and_test_xform(updated_file_bytes, file_ext) - - # Return the updated form as a StreamingResponse - return StreamingResponse( - form_data, - media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - headers={"Content-Disposition": f"attachment; filename={form.filename}"}, - ) + return StreamingResponse( + updated_form, + media_type=( + "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" + ), + headers={"Content-Disposition": "attachment; filename=updated_form.xlsx"}, + ) + else: + await central_crud.validate_and_update_user_xlsform( + xlsform, + task_count=1, # NOTE this must be included to append task_filter choices + ) + return Response(status_code=HTTPStatus.OK) @router.post("/{project_id}/generate-project-data") async def generate_files( background_tasks: BackgroundTasks, - xls_form_upload: Optional[UploadFile] = File(None), + xlsform_upload: Optional[BytesIO] = Depends(central_deps.read_optional_xlsform), + additional_entities: list[str] = None, db: Session = Depends(database.get_db), project_user_dict: ProjectUserDict = Depends(project_manager), ): @@ -718,8 +715,10 @@ async def generate_files( Args: background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically. - xls_form_upload (UploadFile, optional): A custom XLSForm to use in the project. + xlsform_upload (UploadFile, optional): A custom XLSForm to use in the project. A file should be provided if user wants to upload a custom xls form. + additional_entities (list[str]): If additional Entity lists need to be + created (i.e. the project form references multiple geometries). db (Session): Database session, provided automatically. project_user_dict (ProjectUserDict): Project admin role. @@ -728,28 +727,40 @@ async def generate_files( """ project = project_user_dict.get("project") project_id = project.id + form_category = project.xform_category + task_count = len(project.tasks) - log.debug(f"Generating media files tasks for project: {project.id}") + log.debug(f"Generating additional files for project: {project.id}") - custom_xls_form = None - file_ext = ".xls" - if xls_form_upload: - log.debug("Validating uploaded XLS form") + if xlsform_upload: + log.debug("User provided custom XLSForm") - file_path = Path(xls_form_upload.filename) - file_ext = file_path.suffix.lower() - allowed_extensions = {".xls", ".xlsx", ".xml"} - if file_ext not in allowed_extensions: - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, - detail=f"Invalid file extension, must be {allowed_extensions}", - ) + # Validate uploaded form + await central_crud.validate_and_update_user_xlsform( + xlsform=xlsform_upload, + form_category=form_category, + task_count=task_count, + additional_entities=additional_entities, + ) + xlsform = xlsform_upload - custom_xls_form = await xls_form_upload.read() + else: + log.debug(f"Using default XLSForm for category: '{form_category}'") - # Write XLS form content to db - project.form_xls = custom_xls_form - db.commit() + form_filename = XLSFormType(form_category).name + xlsform_path = f"{xlsforms_path}/{form_filename}.xls" + with open(xlsform_path, "rb") as f: + xlsform = BytesIO(f.read()) + + project_xlsform = await central_crud.append_fields_to_user_xlsform( + xlsform=xlsform, + form_category=form_category, + task_count=task_count, + additional_entities=additional_entities, + ) + # Write XLS form content to db + project.form_xls = project_xlsform.getvalue() + db.commit() # Create task in db and return uuid log.debug(f"Creating export background task for project ID: {project_id}") @@ -762,8 +773,6 @@ async def generate_files( project_crud.generate_project_files, db, project_id, - BytesIO(custom_xls_form) if custom_xls_form else None, - file_ext, background_task_id, ) @@ -934,13 +943,6 @@ async def download_form( "Content-Disposition": "attachment; filename=submission_data.xls", "Content-Type": "application/media", } - if not project.form_xls: - form_filename = XLSFormType(project.xform_category).name - xlsform_path = f"{xlsforms_path}/{form_filename}.xls" - if os.path.exists(xlsform_path): - return FileResponse(xlsform_path, filename="form.xls") - else: - raise HTTPException(status_code=404, detail="Form not found") return Response(content=project.form_xls, headers=headers) @@ -948,7 +950,7 @@ async def download_form( async def update_project_form( xform_id: str = Form(...), category: XLSFormType = Form(...), - upload: UploadFile = File(...), + xlsform: BytesIO = Depends(central_deps.read_xlsform), db: Session = Depends(database.get_db), project_user_dict: ProjectUserDict = Depends(project_manager), ) -> project_schemas.ProjectBase: @@ -956,7 +958,6 @@ async def update_project_form( Also updates the category and custom XLSForm data in the database. """ - # TODO migrate most logic to project_crud project = project_user_dict["project"] # TODO we currently do nothing with the provided category @@ -969,38 +970,22 @@ async def update_project_form( # with open(xlsform_path, "rb") as f: # new_xform_data = BytesIO(f.read()) - file_ext = Path(upload.filename or "x.xls").suffix.lower() - allowed_extensions = [".xls", ".xlsx", ".xml"] - if file_ext not in allowed_extensions: - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, - detail="Provide a valid .xls, .xlsx, .xml file.", - ) - new_xform_data = await upload.read() - # Update the XLSForm blob in the database - project.form_xls = new_xform_data - new_xform_data = BytesIO(new_xform_data) - - # TODO related to above info about category updating - # # Update form category in database - # project.xform_category = category - - # Commit changes to db - db.commit() - # Get ODK Central credentials for project odk_creds = await project_deps.get_odk_credentials(db, project.id) # Update ODK Central form data await central_crud.update_project_xform( xform_id, project.odkid, - new_xform_data, - file_ext, + xlsform, category, len(project.tasks), odk_creds, ) + # Commit changes to db + project.form_xls = xlsform.getvalue() + db.commit() + return project diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 5fcce71d12..0af91a8e85 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:dece6ce0db5c1445154361c5c4529b58d558fe30d21e450df8a375de26648ba0" +content_hash = "sha256:a02ce45e28be043d654f512ec57a6db6adf4cba528cb2c937191e3d025ca5490" [[package]] name = "aiohttp" @@ -1579,7 +1579,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.16.1" +version = "0.16.4" requires_python = ">=3.10" summary = "Processing field data from ODK to OpenStreetMap format." dependencies = [ @@ -1605,8 +1605,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.16.1.tar.gz", hash = "sha256:c724c6a8650c1eb62a059317bad46d2f08f612feb4d0ce76dfc2faae90410f13"}, - {file = "osm_fieldwork-0.16.1-py3-none-any.whl", hash = "sha256:c9dbdec0c29747797c63874208fa014a37071b3e663ea63dfd0d3f8ebbe00f69"}, + {file = "osm-fieldwork-0.16.4.tar.gz", hash = "sha256:0285313d3e4bd99df0cccd91b8706b6d3f66ae427bab259250df19b07c51d31b"}, + {file = "osm_fieldwork-0.16.4-py3-none-any.whl", hash = "sha256:595afcf05a0a3fda035e5c2c342b5a5c1bcfa2e21002098f6c670c7e502baf93"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index aeca8c1689..6a5d4b411a 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -45,8 +45,8 @@ dependencies = [ "defusedxml>=0.7.1", "pyjwt>=2.8.0", "async-lru>=2.0.4", + "osm-fieldwork>=0.16.4", "osm-login-python==2.0.0", - "osm-fieldwork==0.16.1", "osm-rawdata==0.3.2", "fmtm-splitter==1.3.0", ] diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index f96bf19a83..dd3ce11ee4 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -385,7 +385,7 @@ const PostFormUpdate = (url: string, projectData: Record) => { const formFormData = new FormData(); formFormData.append('xform_id', projectData.xformId); formFormData.append('category', projectData.category); - formFormData.append('upload', projectData.upload); + formFormData.append('xlsform', projectData.upload); const postFormUpdateResponse = await axios.post(url, formFormData); const resp: ProjectDetailsModel = postFormUpdateResponse.data; @@ -460,7 +460,7 @@ const ValidateCustomForm = (url: string, formUpload: any) => { const validateCustomForm = async (url: any, formUpload: any) => { try { const formUploadFormData = new FormData(); - formUploadFormData.append('form', formUpload); + formUploadFormData.append('xlsform', formUpload); // response is in file format so we need to convert it to blob const getTaskSplittingResponse = await axios.post(url, formUploadFormData, { diff --git a/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx b/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx index 6271f7d4ee..0f44d6b98d 100644 --- a/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx +++ b/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx @@ -4,11 +4,10 @@ import UploadArea from '../../common/UploadArea'; import Button from '../../common/Button'; import { CustomSelect } from '@/components/common/Select'; import CoreModules from '@/shared/CoreModules'; -import { FormCategoryService, ValidateCustomForm } from '@/api/CreateProjectService'; +import { FormCategoryService } from '@/api/CreateProjectService'; +import { DownloadProjectForm } from '@/api/Project'; import { PostFormUpdate } from '@/api/CreateProjectService'; import { CreateProjectActions } from '@/store/slices/CreateProjectSlice'; -import { CommonActions } from '@/store/slices/CommonSlice'; -import { Loader2 } from 'lucide-react'; import useDocumentTitle from '@/utilfunctions/useDocumentTitle'; type FileType = { @@ -28,8 +27,6 @@ const FormUpdateTab = ({ projectId }) => { const xFormId = CoreModules.useAppSelector((state) => state.project.projectInfo.xform_id); const formCategoryList = useAppSelector((state) => state.createproject.formCategoryList); const sortedFormCategoryList = formCategoryList.slice().sort((a, b) => a.title.localeCompare(b.title)); - const customFileValidity = useAppSelector((state) => state.createproject.customFileValidity); - const validateCustomFormLoading = useAppSelector((state) => state.createproject.validateCustomFormLoading); const selectedCategory = useAppSelector((state) => state.createproject.editProjectDetails.xform_category); const formUpdateLoading = useAppSelector((state) => state.createproject.formUpdateLoading); @@ -37,45 +34,16 @@ const FormUpdateTab = ({ projectId }) => { dispatch(FormCategoryService(`${import.meta.env.VITE_API_URL}/central/list-forms`)); }, []); - const validateForm = () => { - setError({ formError: '', categoryError: '' }); - let isValid = true; - if (!uploadForm || (uploadForm && uploadForm?.length === 0)) { - setError((prev) => ({ ...prev, formError: 'Form is required.' })); - isValid = false; - } - if (!customFileValidity && uploadForm && uploadForm.length > 0) { - dispatch( - CommonActions.SetSnackBar({ - open: true, - message: 'Your file is invalid', - variant: 'error', - duration: 2000, - }), - ); - isValid = false; - } - return isValid; - }; - const onSave = () => { - if (validateForm()) { - dispatch( - PostFormUpdate(`${import.meta.env.VITE_API_URL}/projects/update-form?project_id=${projectId}`, { - xformId: xFormId, - category: selectedCategory, - upload: uploadForm && uploadForm?.[0]?.url, - }), - ); - } + dispatch( + PostFormUpdate(`${import.meta.env.VITE_API_URL}/projects/update-form?project_id=${projectId}`, { + xformId: xFormId, + category: selectedCategory, + upload: uploadForm && uploadForm?.[0]?.url, + }), + ); }; - useEffect(() => { - if (uploadForm && uploadForm?.length > 0 && !customFileValidity) { - dispatch(ValidateCustomForm(`${import.meta.env.VITE_API_URL}/projects/validate-form`, uploadForm?.[0]?.url)); - } - }, [uploadForm]); - return (
@@ -103,6 +71,24 @@ const FormUpdateTab = ({ projectId }) => { {`if uploading the final submissions to OSM.`}

+

+ Please{' '} + + dispatch( + DownloadProjectForm( + `${import.meta.env.VITE_API_URL}/projects/download-form/${projectId}/`, + 'form', + projectId, + ), + ) + } + > + download + {' '} + {`your form, modify it, before re-uploading below:`} +

{ data={uploadForm || []} filterKey="url" onUploadFile={(updatedFiles: FileType[]) => { - dispatch(CreateProjectActions.SetCustomFileValidity(false)); setUploadForm(updatedFiles); }} acceptedInput=".xls, .xlsx, .xml" /> - {validateCustomFormLoading && ( -
- -

Validating form...

-
- )} {error.formError &&

{error.formError}

}