Filter data extract with osm-rawdata (allow custom form upload) (#881)

* feat: used osm-rawdata instead of osm-fieldwork for data extracts from osm

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* feat: filter data extract with osm-rawdata

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* feat: added new package osm-rawdata in pyproject.toml

* fix: conflict in pyproject

* fix: conflict in pyproject

* build: update osm-fieldwork --> 0.3.6, osm-rawdata --> 0.1.3

* fix: import osm-fieldwork data_models via helper

---------

Co-authored-by: sujanadh <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Niraj Adhikari <[email protected]>
Co-authored-by: spwoodcock <[email protected]>

src/backend/app/projects/project_crud.py

@@ -41,10 +41,12 @@
 from geojson import dump
 from loguru import logger as log
 from osm_fieldwork import basemapper
+from osm_fieldwork.data_models import data_models_path
+from osm_fieldwork.filter_data import FilterData
 from osm_fieldwork.json2osm import json2osm
-from osm_fieldwork.make_data_extract import PostgresClient
 from osm_fieldwork.OdkCentral import OdkAppUser
 from osm_fieldwork.xlsforms import xlsforms_path
+from osm_rawdata.postgres import PostgresClient
 from shapely import wkt
 from shapely.geometry import (
     LineString,
@@ -1448,8 +1450,6 @@ def generate_appuser_files(
         one = result.first()
 
         if one:
-            prefix = one.project_name_prefix
-
             # Get odk credentials from project.
             odk_credentials = {
                 "odk_central_url": one.odk_central_url,
@@ -1461,44 +1461,51 @@ def generate_appuser_files(
 
             xform_title = one.xform_title if one.xform_title else None
 
+            category = xform_title
             if upload:
-                xlsform = f"/tmp/custom_form.{form_type}"
+                xlsform = f"/tmp/{category}.{form_type}"
                 contents = upload
                 with open(xlsform, "wb") as f:
                     f.write(contents)
             else:
                 xlsform = f"{xlsforms_path}/{xform_title}.xls"
 
-            category = xform_title
-
             # Data Extracts
             if extracts_contents is not None:
                 project_log.info("Uploading data extracts")
                 upload_custom_data_extracts(db, project_id, extracts_contents)
 
             else:
+                project = (
+                    db.query(db_models.DbProject)
+                    .filter(db_models.DbProject.id == project_id)
+                    .first()
+                )
+                config_file_contents = project.form_config_file
+
                 project_log.info("Extracting Data from OSM")
 
-                # OSM Extracts for whole project
-                pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass")
-                # This file will store osm extracts
-                outfile = f"/tmp/{prefix}_{xform_title}.geojson"
+                config_path = "/tmp/config.yaml"
+                if config_file_contents:
+                    with open(config_path, "w", encoding="utf-8") as config_file_handle:
+                        config_file_handle.write(config_file_contents.decode("utf-8"))
+                else:
+                    config_path = f"{data_models_path}/{category}.yaml"
 
+                # # OSM Extracts for whole project
+                pg = PostgresClient("underpass", config_path)
                 outline = json.loads(one.outline)
-                outline_geojson = pg.getFeatures(
-                    boundary=outline,
-                    filespec=outfile,
-                    polygon=extract_polygon,
-                    xlsfile=f"{category}.xls",
-                    category=category,
-                )
+                boundary = {"type": "Feature", "properties": {}, "geometry": outline}
+                data_extract = pg.execQuery(boundary)
+                filter = FilterData(xlsform)
+                filtered_data_extract = filter.cleanData(data_extract)
 
-                updated_outline_geojson = {"type": "FeatureCollection", "features": []}
+                updated_data_extract = {"type": "FeatureCollection", "features": []}
 
                 # Collect feature mappings for bulk insert
                 feature_mappings = []
 
-                for feature in outline_geojson["features"]:
+                for feature in filtered_data_extract["features"]:
                     # If the osm extracts contents do not have a title, provide an empty text for that.
                     feature["properties"]["title"] = ""
 
@@ -1517,9 +1524,8 @@ def generate_appuser_files(
                         "geometry": wkb_element,
                         "properties": feature["properties"],
                     }
-                    updated_outline_geojson["features"].append(feature)
+                    updated_data_extract["features"].append(feature)
                     feature_mappings.append(feature_mapping)
-
                 # Bulk insert the osm extracts into the db.
                 db.bulk_insert_mappings(db_models.DbFeatures, feature_mappings)
 

src/backend/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "python-multipart>=0.0.6",
     "sqlalchemy>=2.0.21",
     "SQLAlchemy-Utils==0.41.1",
-    "psycopg2==2.9.7",
+    "psycopg2>=2.9.7",
     "geoalchemy2==0.14.1",
     "geojson==3.0.1",
     "shapely==2.0.1",
@@ -44,7 +44,8 @@ dependencies = [
     "py-cpuinfo==9.0.0",
     "loguru>=0.7.0",
     "osm-login-python>=1.0.1",
-    "osm-fieldwork==0.3.6rc1",
+    "osm-fieldwork==0.3.6",
+    "osm-rawdata==0.1.3",
 ]
 requires-python = ">=3.10,<3.12"
 readme = "../../README.md"