Merge branch 'staging'

README.md

@@ -174,7 +174,7 @@ ODK Central server, using the
 [ODK Central API](https://odkcentral.docs.apiary.io/#) to allocate specific
 areas/features to individual mappers, and receive their data submissions.
 
-![1](https://github.com/hotosm/fmtm/assets/97789856/305be31a-96b4-42df-96fc-6968e9bd4e5f)
+![image](https://github.com/hotosm/fmtm/assets/97273021/e1bac364-d9dd-47b3-b10f-676e6abab298)
 
 #### Manager Web Interface (with PC browser-friendlymap view)
 
@@ -190,27 +190,42 @@ A computer-screen-optimized web app that allows Campaign Managers to:
 #### Steps to create a project in FMTM
 
 - Go to [fmtm](https://fmtm.hotosm.org/) .
-- If you are new then on the top right cornor click on Sign up and create an account.
-  Else, Sign in to your existing account .
+- Sign into your OpenStreetMap account by clicking the top right button.
 - Click the '+ CREATE NEW PROJECT' button.
-- Enter the project details.
+- Enter the project details like name, description, instructions, and
+  (optional) ODK credentials.
 
-![2](https://github.com/hotosm/fmtm/assets/97789856/97c38c80-aa0e-4fe2-b8a5-f4ee43a9a63a)
+  ![2](https://github.com/hotosm/fmtm/assets/97273021/9b8d7a83-11e3-4623-a14a-66ac47d91443)
 
-- Upload Area in the GEOJSON file format.
+- Draw or Upload Area in the GeoJSON file format.
 
-![3](https://github.com/hotosm/fmtm/assets/97789856/680eb831-790a-48f1-8997-c20b5213909d)
+  ![3](https://github.com/hotosm/fmtm/assets/97273021/892f9051-dbb3-434d-b35a-94e90435a5d8)
 
-- Define the tasks of the project.
+- Choose the category of field mapping. If you require a custom
+  XLSForm to be used, upload it at this point.
 
-![Screenshot 2023-06-07 232152](https://github.com/hotosm/fmtm/assets/97789856/b735a661-d0f6-46b8-b548-5ad7b1928480)
+  ![image](https://github.com/hotosm/fmtm/assets/97273021/02ebb3af-a48f-40eb-b9f8-fd7eec393485)
 
-- Select Form .
+- Generate a data extract from your category selection, or upload your
+  own geometries inside a custom data extract GeoJSON.
 
-![Screenshot 2023-06-07 232316](https://github.com/hotosm/fmtm/assets/97789856/475a6070-4897-4e84-8050-6ecf024d0095)
+  ![image](https://github.com/hotosm/fmtm/assets/97273021/e4a7f9a6-b411-4f8c-b2ba-9fa9e0d42531)
+
+- Choose the type of task splitting:
+
+  - Divide on square: create a simple grid over your project area.
+  - Choose area as tasks: if you have already determined how tasks
+    should be split (your project area contains multiple polygons), then
+    select this option.
+  - Task Splitting Algorithm: use the FMTM splitting algorithm to split
+    task boundaries by average building and linear features.
+
+  ![image](https://github.com/hotosm/fmtm/assets/97273021/c8a905e2-dd52-49e8-87df-0ac6d9a85077)
 
 - Click on Submit button.
 
+  ![image](https://github.com/hotosm/fmtm/assets/97273021/0c95e324-3365-4d0a-ba91-c9f9b06b2318)
+
 #### FMTM Backend
 
 A back end that converts the project parameters entered by the Campaign Manager in

src/backend/app/central/central_crud.py

@@ -20,7 +20,6 @@
 import os
 from asyncio import gather
 from io import BytesIO
-from pathlib import Path
 from typing import Optional, Union
 
 from defusedxml import ElementTree
@@ -450,83 +449,68 @@ async def read_and_test_xform(
         form_file_ext (str): type of form (.xls, .xlsx, or .xml).
         return_form_data (bool): return the XForm data.
     """
-    # TODO xls2xform_convert requires files on disk
-    # TODO create PR to accept BytesIO?
-
     # Read from BytesIO object
-    input_content = input_data.getvalue()
     file_ext = form_file_ext.lower()
 
-    input_path = Path(f"/tmp/fmtm_form_input_tmp{file_ext}")
-    # This file will store xml contents of an xls form
-    # NOTE a file on disk is required by xls2xform_convert
-    output_path = Path("/tmp/fmtm_xform_temp.xml")
-
     if file_ext == ".xml":
-        # Create output file to write to
-        output_path.touch(exist_ok=True)
-        # Write input_content to a temporary file
-        with open(output_path, "wb") as f:
-            f.write(input_content)
+        xform_bytesio = input_data
+        # Parse / validate XForm
+        try:
+            ElementTree.fromstring(xform_bytesio.getvalue())
+        except ElementTree.ParseError as e:
+            log.error(e)
+            msg = f"Error parsing XForm XML: Possible reason: {str(e)}"
+            raise HTTPException(
+                status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
+            ) from e
     else:
-        # Create input file to write to
-        input_path.touch(exist_ok=True)
-        with open(input_path, "wb") as f:
-            f.write(input_content)
         try:
-            log.debug(f"Converting xlsform -> xform: {str(output_path)}")
-            # FIXME should this be validate=True?
-            xls2xform_convert(
-                xlsform_path=str(input_path),
-                xform_path=str(output_path),
+            log.debug("Converting xlsform -> xform")
+            # NOTE do not enable validate=True, as this requires Java installed
+            xform_bytesio, warnings = xls2xform_convert(
+                xlsform_path=f"/tmp/form{file_ext}",
                 validate=False,
+                xlsform_object=input_data,
             )
+            if warnings:
+                log.warning(warnings)
         except Exception as e:
             log.error(e)
-            msg = f"XLSForm is invalid. Possible reason: {str(e)}"
+            msg = f"XLSForm is invalid: {str(e)}"
             raise HTTPException(
                 status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
             ) from e
 
-    # Parse XForm
-    try:
-        # TODO for memory object use ElementTree.fromstring()
-        xml_parsed = ElementTree.parse(str(output_path))
-        if return_form_data:
-            xml_bytes = ElementTree.tostring(xml_parsed.getroot())
-            return BytesIO(xml_bytes)
-    except ElementTree.ParseError as e:
-        log.error(e)
-        msg = f"Error parsing XForm XML: Possible reason: {str(e)}"
-        raise HTTPException(
-            status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
-        ) from e
-
-    # Delete temp files
-    if file_ext != ".xml":
-        input_path.unlink()
-    output_path.unlink()
-
+    # Return immediately
     if return_form_data:
-        return xml_parsed
+        return xform_bytesio
+
+    # Load XML
+    xform_xml = ElementTree.fromstring(xform_bytesio.getvalue())
 
     # Extract geojson filenames
     try:
-        root = xml_parsed.getroot()
         namespaces = {"xforms": "http://www.w3.org/2002/xforms"}
         geojson_list = [
             os.path.splitext(inst.attrib["src"].split("/")[-1])[0]
-            for inst in root.findall(".//xforms:instance[@src]", namespaces)
+            for inst in xform_xml.findall(".//xforms:instance[@src]", namespaces)
             if inst.attrib.get("src", "").endswith(".geojson")
         ]
 
+        # No select_one_from_file defined
+        if not geojson_list:
+            msg = (
+                "The form has no select_one_from_file or "
+                "select_multiple_from_file field defined."
+            )
+            raise ValueError(msg) from None
+
         return {"required_media": geojson_list, "message": "Your form is valid"}
 
     except Exception as e:
         log.error(e)
-        msg = f"Error extracting geojson filename: Possible reason: {str(e)}"
         raise HTTPException(
-            status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
+            status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=str(e)
         ) from e
 
 

src/backend/app/submissions/submission_crud.py

@@ -820,3 +820,28 @@ async def get_submission_by_task(
     count = data.get("@odata.count", 0)
 
     return submissions, count
+
+
+async def get_submission_detail(
+    project: db_models.DbProject,
+    task_id: int,
+    submission_id: str,
+    db: Session,
+):
+    """Get the details of a submission.
+
+    Args:
+        project: The project object representing the project.
+        task_id: The ID of the task associated with the submission.
+        submission_id: The ID of the submission.
+        db: The database session.
+
+    Returns:
+        The details of the submission as a JSON object.
+    """
+    odk_credentials = await project_deps.get_odk_credentials(db, project.id)
+    odk_form = get_odk_form(odk_credentials)
+    xform = f"{project.project_name_prefix}_task_{task_id}"
+    submission = odk_form.getSubmissions(project.odkid, xform, submission_id)
+
+    return json.loads(submission)

src/backend/app/submissions/submission_routes.py

@@ -413,6 +413,7 @@ async def task_submissions(
     project: db_models.DbProject = Depends(project_deps.get_project_by_id),
     page: int = Query(1, ge=1),
     limit: int = Query(13, le=100),
+    submission_id: Optional[str] = None,
     submitted_by: Optional[str] = None,
     review_state: Optional[str] = None,
     submitted_date: Optional[str] = Query(
@@ -463,4 +464,10 @@ async def task_submissions(
         results=data,
         pagination=submission_schemas.PaginationInfo(**pagination.model_dump()),
     )
+    if submission_id:
+        submission_detail = await submission_crud.get_submission_detail(
+            project, task_id, submission_id, db
+        )
+        response = submission_detail.get("value", [])[0]
+
     return response

src/backend/pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "geoalchemy2==0.14.2",
     "geojson==3.1.0",
     "shapely==2.0.2",
-    "pyxform==1.12.2",
+    "pyxform @ git+https://github.com/hotosm/pyxform@feat/xls2xform_convert_bytesio",
     "sentry-sdk==1.38.0",
     "py-cpuinfo==9.0.0",
     "loguru==0.7.2",
@@ -47,7 +47,7 @@ dependencies = [
     "cryptography>=42.0.1",
     "defusedxml>=0.7.1",
     "osm-login-python==1.0.1",
-    "osm-fieldwork==0.5.2",
+    "osm-fieldwork==0.5.3",
     "osm-rawdata==0.2.3",
     "fmtm-splitter==1.1.2",
 ]

src/frontend/package.json

@@ -64,6 +64,19 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@reduxjs/toolkit": "^1.9.1",
     "@sentry/react": "^7.59.3",
+    "@tiptap/extension-bullet-list": "^2.2.4",
+    "@tiptap/extension-document": "^2.2.4",
+    "@tiptap/extension-heading": "^2.2.4",
+    "@tiptap/extension-image": "^2.2.4",
+    "@tiptap/extension-link": "^2.2.4",
+    "@tiptap/extension-list-item": "^2.2.4",
+    "@tiptap/extension-ordered-list": "^2.2.4",
+    "@tiptap/extension-paragraph": "^2.2.4",
+    "@tiptap/extension-text": "^2.2.4",
+    "@tiptap/extension-youtube": "^2.2.4",
+    "@tiptap/pm": "^2.2.4",
+    "@tiptap/react": "^2.2.4",
+    "@tiptap/starter-kit": "^2.2.4",
     "@types/jest": "^29.5.3",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",