From 3bb66ee2f9d0997f1c6504b6912fcdc663280e93 Mon Sep 17 00:00:00 2001 From: sujanadh Date: Thu, 5 Oct 2023 13:38:01 +0545 Subject: [PATCH 1/3] feat: used osm-rawdata instead of osm-fieldwork for data extracts from osm --- src/backend/app/projects/project_crud.py | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index aa28ce322e..a2323164a6 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -42,7 +42,7 @@ from loguru import logger as log from osm_fieldwork import basemapper from osm_fieldwork.json2osm import json2osm -from osm_fieldwork.make_data_extract import PostgresClient +from osm_rawdata.postgres import PostgresClient from osm_fieldwork.OdkCentral import OdkAppUser from osm_fieldwork.xlsforms import xlsforms_path from shapely import wkt @@ -1477,21 +1477,26 @@ def generate_appuser_files( upload_custom_data_extracts(db, project_id, extracts_contents) else: - project_log.info("Extracting Data from OSM") + import osm_fieldwork as of + rootdir = of.__path__[0] + + project = db.query(db_models.DbProject).filter(db_models.DbProject.id == project_id).first() + config_file_contents = project.form_config_file - # OSM Extracts for whole project - pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass") - # This file will store osm extracts - outfile = f"/tmp/{prefix}_{xform_title}.geojson" + project_log.info("Extracting Data from OSM") + + config_path = "/tmp/config.yaml" + if config_file_contents: + with open(config_path, "w",encoding="utf-8") as config_file_handle: + config_file_handle.write(config_file_contents.decode("utf-8")) + else: + config_path = f"{rootdir}/data_models/{category}.yaml" + # # OSM Extracts for whole project + pg = PostgresClient("underpass", config_path) outline = json.loads(one.outline) - outline_geojson = pg.getFeatures( - boundary=outline, - filespec=outfile, - polygon=extract_polygon, - xlsfile=f"{category}.xls", - category=category, - ) + boundary = {"type":"Feature","properties":{},"geometry":outline} + outline_geojson = pg.execQuery(boundary) updated_outline_geojson = {"type": "FeatureCollection", "features": []} @@ -1519,7 +1524,6 @@ def generate_appuser_files( } updated_outline_geojson["features"].append(feature) feature_mappings.append(feature_mapping) - # Bulk insert the osm extracts into the db. db.bulk_insert_mappings(db_models.DbFeatures, feature_mappings) From e1d251e2e4d754b3d651f32153d6e8324465b546 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 08:01:03 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/backend/app/projects/project_crud.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index a2323164a6..d66eb90a30 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -42,9 +42,9 @@ from loguru import logger as log from osm_fieldwork import basemapper from osm_fieldwork.json2osm import json2osm -from osm_rawdata.postgres import PostgresClient from osm_fieldwork.OdkCentral import OdkAppUser from osm_fieldwork.xlsforms import xlsforms_path +from osm_rawdata.postgres import PostgresClient from shapely import wkt from shapely.geometry import ( LineString, @@ -1448,7 +1448,6 @@ def generate_appuser_files( one = result.first() if one: - prefix = one.project_name_prefix # Get odk credentials from project. odk_credentials = { @@ -1478,16 +1477,21 @@ def generate_appuser_files( else: import osm_fieldwork as of + rootdir = of.__path__[0] - project = db.query(db_models.DbProject).filter(db_models.DbProject.id == project_id).first() + project = ( + db.query(db_models.DbProject) + .filter(db_models.DbProject.id == project_id) + .first() + ) config_file_contents = project.form_config_file project_log.info("Extracting Data from OSM") - + config_path = "/tmp/config.yaml" if config_file_contents: - with open(config_path, "w",encoding="utf-8") as config_file_handle: + with open(config_path, "w", encoding="utf-8") as config_file_handle: config_file_handle.write(config_file_contents.decode("utf-8")) else: config_path = f"{rootdir}/data_models/{category}.yaml" @@ -1495,7 +1499,7 @@ def generate_appuser_files( # # OSM Extracts for whole project pg = PostgresClient("underpass", config_path) outline = json.loads(one.outline) - boundary = {"type":"Feature","properties":{},"geometry":outline} + boundary = {"type": "Feature", "properties": {}, "geometry": outline} outline_geojson = pg.execQuery(boundary) updated_outline_geojson = {"type": "FeatureCollection", "features": []} From dd121d7b645ef509a78c5ef406da1b9e5d65badb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 06:08:24 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/backend/app/projects/project_crud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index d66eb90a30..e971e905c1 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -1448,7 +1448,6 @@ def generate_appuser_files( one = result.first() if one: - # Get odk credentials from project. odk_credentials = { "odk_central_url": one.odk_central_url,