From 1eb7ed5819b784ef4e76de6e2044ea4e5f0e2620 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 12:25:32 +0000 Subject: [PATCH 1/9] fix: remove Field(Form()) from schemas --- .../app/organisations/organisation_schemas.py | 12 ++++-------- src/backend/app/projects/project_schemas.py | 10 +++------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index 5e10fa5002..1b50bd0edc 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -34,13 +34,9 @@ class OrganisationIn(ODKCentralIn): """Organisation to create from user input.""" - name: str = Field(Form(..., description="Organisation name")) - description: Optional[str] = Field( - Form(None, description="Organisation description") - ) - url: Optional[HttpUrlStr] = Field( - Form(None, description="Organisation website URL") - ) + name: str + description: Optional[str] = None + url: Optional[HttpUrlStr] = None @computed_field @property @@ -59,7 +55,7 @@ class OrganisationEdit(OrganisationIn): """Organisation to edit via user input.""" # Override to make name optional - name: Optional[str] = Field(Form(None, description="Organisation name")) + name: Optional[str] = None class OrganisationOut(BaseModel): diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index b407933e27..5ab4fd48e5 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -41,13 +41,9 @@ class ODKCentralIn(BaseModel): """ODK Central credentials inserted to database.""" - odk_central_url: Optional[HttpUrlStr] = Field( - Form(None, description="ODK Central URL") - ) - odk_central_user: Optional[str] = Field(Form(None, description="ODK Central User")) - odk_central_password: Optional[str] = Field( - Form(None, description="ODK Central Password") - ) + odk_central_url: Optional[HttpUrlStr] = None + odk_central_user: Optional[str] = None + odk_central_password: Optional[str] = None @field_validator("odk_central_url", mode="after") @classmethod From 70bb901cf84516e6777b9443ec0953705c73f0c4 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 15:12:40 +0000 Subject: [PATCH 2/9] refactor: move geometry generation to project_schemas --- src/backend/app/db/postgis_utils.py | 134 ++++++++++++++---- .../app/organisations/organisation_schemas.py | 3 +- src/backend/app/projects/project_crud.py | 131 ++++------------- src/backend/app/projects/project_routes.py | 29 ++-- src/backend/app/projects/project_schemas.py | 47 +++++- 5 files changed, 191 insertions(+), 153 deletions(-) diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index 4fafaacaa6..237e81ccb1 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -18,17 +18,18 @@ """PostGIS and geometry handling helper funcs.""" import datetime +import json import logging -from json import dumps as json_dumps from typing import Optional, Union +import geojson +import requests from fastapi import HTTPException from geoalchemy2 import WKBElement -from geoalchemy2.shape import to_shape -from geojson import FeatureCollection -from geojson import loads as geojson_loads -from geojson_pydantic import Feature -from shapely.geometry import mapping +from geoalchemy2.shape import from_shape, to_shape +from geojson_pydantic import Feature, Polygon +from geojson_pydantic import FeatureCollection as FeatCol +from shapely.geometry import mapping, shape from sqlalchemy import text from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import Session @@ -78,14 +79,46 @@ def get_centroid( return {} +def geojson_to_geometry( + geojson: Union[FeatCol, Feature, Polygon], +) -> Optional[WKBElement]: + """Convert GeoJSON to SQLAlchemy geometry.""" + parsed_geojson = parse_and_filter_geojson(geojson.model_dump_json(), filter=False) + + if not parsed_geojson: + return None + + features = parsed_geojson.get("features", []) + + if len(features) > 1: + # TODO code to merge all geoms into multipolygon + # TODO do not use convex hull + pass + + geometry = features[0].get("geometry") + + shapely_geom = shape(geometry) + return from_shape(shapely_geom) + + +def read_wkb(wkb: WKBElement): + """Load a WKBElement and return a shapely geometry.""" + return to_shape(wkb) + + +def write_wkb(shape): + """Load shapely geometry and output WKBElement.""" + return from_shape(shape) + + async def geojson_to_flatgeobuf( - db: Session, geojson: FeatureCollection + db: Session, geojson: geojson.FeatureCollection ) -> Optional[bytes]: """From a given FeatureCollection, return a memory flatgeobuf obj. Args: db (Session): SQLAlchemy db session. - geojson (FeatureCollection): a geojson.FeatureCollection object. + geojson (geojson.FeatureCollection): a FeatureCollection object. Returns: flatgeobuf (bytes): a Python bytes representation of a flatgeobuf file. @@ -142,7 +175,7 @@ async def geojson_to_flatgeobuf( FROM (SELECT * FROM public.temp_features as geoms) AS fgb_data; """ # Run the SQL - result = db.execute(text(sql), {"geojson": json_dumps(geojson)}) + result = db.execute(text(sql), {"geojson": json.dumps(geojson)}) # Get a memoryview object, then extract to Bytes flatgeobuf = result.first() @@ -158,7 +191,7 @@ async def geojson_to_flatgeobuf( async def flatgeobuf_to_geojson( db: Session, flatgeobuf: bytes -) -> Optional[FeatureCollection]: +) -> Optional[geojson.FeatureCollection]: """Converts FlatGeobuf data to GeoJSON. Args: @@ -166,7 +199,7 @@ async def flatgeobuf_to_geojson( flatgeobuf (bytes): FlatGeobuf data in bytes format. Returns: - FeatureCollection: A GeoJSON FeatureCollection object. + geojson.FeatureCollection: A FeatureCollection object. """ sql = text( """ @@ -204,41 +237,49 @@ async def flatgeobuf_to_geojson( return None if feature_collection: - return geojson_loads(json_dumps(feature_collection[0])) + return geojson.loads(json.dumps(feature_collection[0])) return None -async def parse_and_filter_geojson(geojson_str: str) -> Optional[FeatureCollection]: +def parse_and_filter_geojson( + geojson_str: str, filter: bool = True +) -> Optional[geojson.FeatureCollection]: """Parse geojson string and filter out incomaptible geometries.""" - log.debug("Parsing geojson file") - geojson_parsed = geojson_loads(geojson_str) - if isinstance(geojson_parsed, FeatureCollection): + log.debug("Parsing geojson string") + geojson_parsed = geojson.loads(geojson_str) + if isinstance(geojson_parsed, geojson.FeatureCollection): log.debug("Already in FeatureCollection format, skipping reparse") featcol = geojson_parsed - elif isinstance(geojson_parsed, Feature): + elif isinstance(geojson_parsed, geojson.Feature): log.debug("Converting Feature to FeatureCollection") - featcol = FeatureCollection(geojson_parsed) + featcol = geojson.FeatureCollection(features=[geojson_parsed]) else: log.debug("Converting geometry to FeatureCollection") - featcol = FeatureCollection[Feature(geometry=geojson_parsed)] + featcol = geojson.FeatureCollection( + features=[geojson.Feature(geometry=geojson_parsed)] + ) - # Validating Coordinate Reference System - check_crs(featcol) + # Exit early if no geoms + if not featcol.get("features", []): + return None - geom_type = await get_featcol_main_geom_type(featcol) + # Return unfiltered featcol + if not filter: + return featcol # Filter out geoms not matching main type + geom_type = get_featcol_main_geom_type(featcol) features_filtered = [ feature for feature in featcol.get("features", []) if feature.get("geometry", {}).get("type", "") == geom_type ] - return FeatureCollection(features_filtered) + return geojson.FeatureCollection(features_filtered) -async def get_featcol_main_geom_type(featcol: FeatureCollection) -> str: +def get_featcol_main_geom_type(featcol: geojson.FeatureCollection) -> str: """Get the predominant geometry type in a FeatureCollection.""" geometry_counts = {"Polygon": 0, "Point": 0, "Polyline": 0} @@ -250,7 +291,7 @@ async def get_featcol_main_geom_type(featcol: FeatureCollection) -> str: return max(geometry_counts, key=geometry_counts.get) -async def check_crs(input_geojson: Union[dict, FeatureCollection]): +async def check_crs(input_geojson: Union[dict, geojson.FeatureCollection]): """Validate CRS is valid for a geojson.""" log.debug("validating coordinate reference system") @@ -305,3 +346,46 @@ def is_valid_coordinate(coord): if not is_valid_coordinate(first_coordinate): log.error(error_message) raise HTTPException(status_code=400, detail=error_message) + + +def get_address_from_lat_lon(latitude, longitude): + """Get address using Nominatim, using lat,lon.""" + base_url = "https://nominatim.openstreetmap.org/reverse" + + params = { + "format": "json", + "lat": latitude, + "lon": longitude, + "zoom": 18, + } + headers = {"Accept-Language": "en"} # Set the language to English + + log.debug("Getting Nominatim address from project centroid") + response = requests.get(base_url, params=params, headers=headers) + if (status_code := response.status_code) != 200: + log.error(f"Getting address string failed: {status_code}") + return None + + data = response.json() + log.debug(f"Nominatim response: {data}") + + address = data.get("address", None) + if not address: + log.error(f"Getting address string failed: {status_code}") + return None + + country = address.get("country", "") + city = address.get("city", "") + + address_str = f"{city},{country}" + + if not address_str or address_str == ",": + log.error("Getting address string failed") + return None + + return address_str + + +async def get_address_from_lat_lon_async(latitude, longitude): + """Async wrapper for get_address_from_lat_lon.""" + return get_address_from_lat_lon(latitude, longitude) diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index 1b50bd0edc..aa12edbdc6 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -20,8 +20,7 @@ from re import sub from typing import Optional -from fastapi import Form -from pydantic import BaseModel, Field, computed_field +from pydantic import BaseModel, computed_field from app.config import HttpUrlStr from app.models.enums import OrganisationType diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 670bd6ea35..8786522262 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -31,7 +31,6 @@ import requests import shapely.wkb as wkblib import sozipfile.sozipfile as zipfile -import sqlalchemy from asgiref.sync import async_to_sync from fastapi import File, HTTPException, Response, UploadFile from fastapi.concurrency import run_in_threadpool @@ -62,6 +61,7 @@ flatgeobuf_to_geojson, geojson_to_flatgeobuf, geometry_to_geojson, + get_address_from_lat_lon_async, get_featcol_main_geom_type, parse_and_filter_geojson, ) @@ -286,7 +286,7 @@ async def create_project_with_project_info( author_id=current_user.id, odkid=odk_project_id, project_name_prefix=project_name, - **project_metadata.model_dump(exclude=["project_info"]), + **project_metadata.model_dump(exclude=["project_info", "outline_geojson"]), ) db.add(db_project) @@ -334,39 +334,23 @@ async def upload_xlsform( raise HTTPException(status_code=400, detail={"message": str(e)}) from e -async def update_multi_polygon_project_boundary( +async def create_tasks_from_geojson( db: Session, project_id: int, boundary: str, ): - """Update the boundary for a project & update tasks. - - TODO requires refactoring, as it has too large of - a scope. It should update a project boundary only, then manage - tasks in another function. - - This function receives the project_id and boundary as a parameter - and creates a task for each polygon in the database. - This function also creates a project outline from the multiple - polygons received. - """ + """Create tasks for a project, from provided task boundaries.""" try: if isinstance(boundary, str): boundary = json.loads(boundary) - # verify project exists in db - db_project = await get_project_by_id(db, project_id) - if not db_project: - log.error(f"Project {project_id} doesn't exist!") - return False - # Update the boundary polyon on the database. if boundary["type"] == "Feature": polygons = [boundary] else: polygons = boundary["features"] log.debug(f"Processing {len(polygons)} task geometries") - for polygon in polygons: + for index, polygon in enumerate(polygons): # If the polygon is a MultiPolygon, convert it to a Polygon if polygon["geometry"]["type"] == "MultiPolygon": log.debug("Converting MultiPolygon to Polygon") @@ -375,49 +359,21 @@ async def update_multi_polygon_project_boundary( 0 ] - # def remove_z_dimension(coord): - # """Helper to remove z dimension. - - # To be used in lambda, to remove z dimension from - # each coordinate in the feature's geometry. - # """ - # return coord.pop() if len(coord) == 3 else None - - # # Apply the lambda function to each coordinate in its geometry - # list(map(remove_z_dimension, polygon["geometry"]["coordinates"][0])) - db_task = db_models.DbTask( project_id=project_id, outline=wkblib.dumps(shape(polygon["geometry"]), hex=True), - project_task_index=1, + project_task_index=index, ) db.add(db_task) - db.commit() - - # Id is passed in the task_name too - db_task.project_task_name = str(db_task.id) log.debug( "Created database task | " f"Project ID {project_id} | " - f"Task ID {db_task.project_task_name}" + f"Task index {index}" ) - db.commit() - - # Generate project outline from tasks - query = text( - f"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline))) - FROM tasks - WHERE project_id={project_id};""" - ) - - log.debug("Generating project outline from tasks") - result = db.execute(query) - data = result.fetchone() - - await update_project_location_info(db_project, data[0]) + # Commit all tasks and update project location in db db.commit() - db.refresh(db_project) + log.debug("COMPLETE: creating project boundary, based on task boundaries") return True @@ -570,7 +526,11 @@ async def split_geojson_into_tasks( async def update_project_boundary( db: Session, project_id: int, boundary: str, meters: int ): - """Update the boundary for a project and update tasks.""" + """Update the boundary for a project and update tasks. + + TODO this needs a big refactor / removal + # + """ # verify project exists in db db_project = await get_project_by_id(db, project_id) if not db_project: @@ -617,7 +577,12 @@ def remove_z_dimension(coord): else: outline = shape(features[0]["geometry"]) - await update_project_location_info(db_project, outline.wkt) + centroid = (wkt.loads(outline.wkt)).centroid.wkt + db_project.centroid = centroid + geometry = wkt.loads(centroid) + longitude, latitude = geometry.x, geometry.y + address = await get_address_from_lat_lon_async(latitude, longitude) + db_project.location_str = address if address is not None else "" db.commit() db.refresh(db_project) @@ -994,15 +959,16 @@ async def upload_custom_data_extract( if not project: raise HTTPException(status_code=404, detail="Project not found") - featcol_filtered = await parse_and_filter_geojson(geojson_str) + featcol_filtered = parse_and_filter_geojson(geojson_str) if not featcol_filtered: raise HTTPException( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail="Could not process geojson input", ) + await check_crs(featcol_filtered) # Get geom type from data extract - geom_type = await get_featcol_main_geom_type(featcol_filtered) + geom_type = get_featcol_main_geom_type(featcol_filtered) if geom_type not in ["Polygon", "Polyline", "Point"]: msg = ( "Extract does not contain valid geometry types, from 'Polygon' " @@ -1241,6 +1207,7 @@ def generate_appuser_files( odk_credentials = project_schemas.ODKCentralDecrypted(**odk_credentials) if custom_form: + log.debug("User provided custom XLSForm") # TODO uncomment after refactor to use BytesIO # xlsform = custom_form @@ -1248,6 +1215,8 @@ def generate_appuser_files( with open(xlsform, "wb") as f: f.write(custom_form.getvalue()) else: + log.debug(f"Using default XLSForm for category {form_category}") + # TODO uncomment after refactor to use BytesIO # xlsform_path = f"{xlsforms_path}/{form_category}.xls" # with open(xlsform_path, "rb") as f: @@ -1265,6 +1234,7 @@ def generate_appuser_files( # FIXME do we need these geoms in the db? # FIXME can we remove this section? + log.debug("Adding data extract geometries to database") get_extract_geojson_sync = async_to_sync(get_project_features_geojson) data_extract_geojson = get_extract_geojson_sync(db, project_id) # Collect feature mappings for bulk insert @@ -1598,7 +1568,7 @@ async def convert_to_project_feature(db_project_feature: db_models.DbFeatures): TODO refactor to use Pydantic model methods instead. """ if db_project_feature: - app_project_feature: project_schemas.Feature = db_project_feature + app_project_feature: project_schemas.GeojsonFeature = db_project_feature if db_project_feature.geometry: app_project_feature.geometry = geometry_to_geojson( @@ -1614,7 +1584,7 @@ async def convert_to_project_feature(db_project_feature: db_models.DbFeatures): async def convert_to_project_features( db_project_features: List[db_models.DbFeatures], -) -> List[project_schemas.Feature]: +) -> List[project_schemas.GeojsonFeature]: """Legacy function to convert db models --> Pydantic. TODO refactor to use Pydantic model methods instead. @@ -2042,49 +2012,6 @@ async def convert_geojson_to_osm(geojson_file: str): return json2osm(geojson_file) -async def get_address_from_lat_lon(latitude, longitude): - """Get address using Nominatim, using lat,lon.""" - base_url = "https://nominatim.openstreetmap.org/reverse" - - params = { - "format": "json", - "lat": latitude, - "lon": longitude, - "zoom": 18, - } - headers = {"Accept-Language": "en"} # Set the language to English - - response = requests.get(base_url, params=params, headers=headers) - data = response.json() - address = data["address"]["country"] - - if response.status_code == 200: - if "city" in data["address"]: - city = data["address"]["city"] - address = f"{city}" + "," + address - return address - else: - return "Address not found." - - -async def update_project_location_info( - db_project: sqlalchemy.orm.declarative_base, project_boundary: str -): - """Update project boundary, centroid, address. - - Args: - db_project(sqlalchemy.orm.declarative_base): The project database record. - project_boundary(str): WKT string geometry. - """ - db_project.outline = project_boundary - centroid = (wkt.loads(project_boundary)).centroid.wkt - db_project.centroid = centroid - geometry = wkt.loads(centroid) - longitude, latitude = geometry.x, geometry.y - address = await get_address_from_lat_lon(latitude, longitude) - db_project.location_str = address if address is not None else "" - - async def get_tasks_count(db: Session, project_id: int): """Get number of tasks for a project.""" db_task = ( diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 45796b548d..fe2d76d97b 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -388,16 +388,16 @@ async def upload_custom_xls( return {"xform_title": f"{category}"} -@router.post("/{project_id}/custom_task_boundaries") -async def upload_custom_task_boundaries( +@router.post("/{project_id}/upload-task-boundaries") +async def upload_project_task_boundaries( project_id: int, - project_geojson: UploadFile = File(...), + task_geojson: UploadFile = File(...), db: Session = Depends(database.get_db), org_user_dict: db_models.DbUser = Depends(org_admin), ): - """Set project task boundaries manually using multi-polygon GeoJSON. + """Set project task boundaries using split GeoJSON from frontend. - Each polygon in the uploaded geojson are made a single task. + Each polygon in the uploaded geojson are made into single task. Required Parameters: project_id (id): ID for associated project. @@ -408,21 +408,14 @@ async def upload_custom_task_boundaries( """ log.debug(f"Uploading project boundary multipolygon for project ID: {project_id}") # read entire file - content = await project_geojson.read() - boundary = json.loads(content) + content = await task_geojson.read() + task_boundaries = json.loads(content) # Validatiing Coordinate Reference System - await check_crs(boundary) + await check_crs(task_boundaries) log.debug("Creating tasks for each polygon in project") - result = await project_crud.update_multi_polygon_project_boundary( - db, project_id, boundary - ) - - if not result: - raise HTTPException( - status_code=428, detail=f"Project with id {project_id} does not exist" - ) + await project_crud.create_tasks_from_geojson(db, project_id, task_boundaries) # Get the number of tasks in a project task_count = await tasks_crud.get_task_count_in_project(db, project_id) @@ -696,7 +689,9 @@ async def update_project_form( return form_updated -@router.get("/{project_id}/features", response_model=list[project_schemas.Feature]) +@router.get( + "/{project_id}/features", response_model=list[project_schemas.GeojsonFeature] +) async def get_project_features( project_id: int, task_id: int = None, diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 5ab4fd48e5..f3a8d9d819 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -22,8 +22,7 @@ from typing import Any, List, Optional, Union from dateutil import parser -from fastapi import Form -from geojson_pydantic import Feature as GeojsonFeature +from geojson_pydantic import Feature, FeatureCollection, Polygon from loguru import logger as log from pydantic import BaseModel, Field, computed_field from pydantic.functional_serializers import field_serializer @@ -32,7 +31,13 @@ from app.config import HttpUrlStr, decrypt_value, encrypt_value from app.db import db_models -from app.db.postgis_utils import geometry_to_geojson +from app.db.postgis_utils import ( + geojson_to_geometry, + geometry_to_geojson, + get_address_from_lat_lon, + read_wkb, + write_wkb, +) from app.models.enums import ProjectPriority, ProjectStatus, TaskSplitType from app.tasks import tasks_schemas from app.users.user_schemas import User @@ -136,9 +141,37 @@ class ProjectIn(BaseModel): task_split_dimension: Optional[int] = None task_num_buildings: Optional[int] = None data_extract_type: Optional[str] = None + outline_geojson: Union[FeatureCollection, Feature, Polygon] # city: str # country: str + @computed_field + @property + def outline(self) -> Optional[Any]: + """Compute WKBElement geom from geojson.""" + if not self.outline_geojson: + return None + return geojson_to_geometry(self.outline_geojson) + + @computed_field + @property + def centroid(self) -> Optional[Any]: + """Compute centroid for project outline.""" + if not self.outline: + return None + return write_wkb(read_wkb(self.outline).centroid) + + @computed_field + @property + def location_str(self) -> Optional[str]: + """Compute geocoded location string from centroid.""" + if not self.centroid: + return None + geom = read_wkb(self.centroid) + latitude, longitude = geom.y, geom.x + address = get_address_from_lat_lon(latitude, longitude) + return address if address is not None else "" + @field_validator("hashtags", mode="after") @classmethod def prepend_hash_to_tags(cls, hashtags: List[str]) -> Optional[List[str]]: @@ -172,11 +205,11 @@ class ProjectUpdate(ProjectIn): organisation_id: Optional[int] = None -class Feature(BaseModel): +class GeojsonFeature(BaseModel): """Features used for Task definitions.""" id: int - geometry: Optional[GeojsonFeature] = None + geometry: Optional[Feature] = None class ProjectSummary(BaseModel): @@ -260,8 +293,8 @@ class ProjectBase(BaseModel): @computed_field @property - def outline_geojson(self) -> Optional[GeojsonFeature]: - """Sanitise the organisation name for use in a URL.""" + def outline_geojson(self) -> Optional[Feature]: + """Compute the geojson outline from WKBElement outline.""" if not self.outline: return None return geometry_to_geojson(self.outline, {"id": self.id}, self.id) From 5542b452a3c8b47680406b6e69bf85e9a0da2025 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 16:23:53 +0000 Subject: [PATCH 3/9] fix(frontend): send project outline geojson during create --- src/frontend/src/components/editproject/UpdateProjectArea.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/editproject/UpdateProjectArea.tsx b/src/frontend/src/components/editproject/UpdateProjectArea.tsx index 12f0a69c38..745d8213ee 100644 --- a/src/frontend/src/components/editproject/UpdateProjectArea.tsx +++ b/src/frontend/src/components/editproject/UpdateProjectArea.tsx @@ -41,7 +41,7 @@ const UpdateProjectArea = ({ projectId }) => { const generateTasksOnMap = () => { dispatch( - GetDividedTaskFromGeojson(`${import.meta.env.VITE_API_URL}/projects/preview_split_by_square/`, { + GetDividedTaskFromGeojson(`${import.meta.env.VITE_API_URL}/projects/preview-split-by-square/`, { geojson: uploadAOI, dimension: projectBoundaryDetails?.dimension, }), From 47fb2175d69e87f972c2de194e7e6d8713c89740 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 16:25:50 +0000 Subject: [PATCH 4/9] fix: only send custom data extracts to splitter --- src/backend/app/projects/project_crud.py | 16 +-- src/backend/app/projects/project_routes.py | 4 +- src/frontend/src/api/CreateProjectService.ts | 118 +++++++++--------- .../createnewproject/SplitTasks.tsx | 29 +++-- 4 files changed, 84 insertions(+), 83 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 8786522262..dfb443c500 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -337,18 +337,18 @@ async def upload_xlsform( async def create_tasks_from_geojson( db: Session, project_id: int, - boundary: str, + boundaries: str, ): """Create tasks for a project, from provided task boundaries.""" try: - if isinstance(boundary, str): - boundary = json.loads(boundary) + if isinstance(boundaries, str): + boundaries = json.loads(boundaries) # Update the boundary polyon on the database. - if boundary["type"] == "Feature": - polygons = [boundary] + if boundaries["type"] == "Feature": + polygons = [boundaries] else: - polygons = boundary["features"] + polygons = boundaries["features"] log.debug(f"Processing {len(polygons)} task geometries") for index, polygon in enumerate(polygons): # If the polygon is a MultiPolygon, convert it to a Polygon @@ -492,7 +492,7 @@ async def split_geojson_into_tasks( db: Session, project_geojson: Union[dict, FeatureCollection], no_of_buildings: int, - extract_geojson: Optional[Union[dict, FeatureCollection]] = None, + extract_geojson: Optional[FeatureCollection] = None, ): """Splits a project into tasks. @@ -502,7 +502,7 @@ async def split_geojson_into_tasks( boundary. extract_geojson (Union[dict, FeatureCollection]): A GeoJSON of the project boundary osm data extract (features). - extract_geojson (Union[dict, FeatureCollection]): A GeoJSON of the project + extract_geojson (FeatureCollection): A GeoJSON of the project boundary osm data extract (features). If not included, an extract is generated automatically. no_of_buildings (int): The number of buildings to include in each task. diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index fe2d76d97b..8e4ce631c0 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -401,7 +401,7 @@ async def upload_project_task_boundaries( Required Parameters: project_id (id): ID for associated project. - project_geojson (UploadFile): Multi-polygon GeoJSON file. + task_geojson (UploadFile): Multi-polygon GeoJSON file. Returns: dict: JSON containing success message, project ID, and number of tasks. @@ -787,7 +787,7 @@ async def get_categories(current_user: AuthUser = Depends(login_required)): return categories -@router.post("/preview_split_by_square/") +@router.post("/preview-split-by-square/") async def preview_split_by_square( project_geojson: UploadFile = File(...), dimension: int = Form(100) ): diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index 484a48ee53..a2826403b2 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -12,8 +12,8 @@ import { task_split_type } from '@/types/enums'; const CreateProjectService: Function = ( url: string, - payload: any, - fileUpload: any, + projectJson: any, + taskAreaGeojson: any, formUpload: any, dataExtractFile: any, ) => { @@ -21,27 +21,21 @@ const CreateProjectService: Function = ( dispatch(CreateProjectActions.CreateProjectLoading(true)); dispatch(CommonActions.SetLoading(true)); - const postCreateProjectDetails = async (url, payload, fileUpload, formUpload) => { + const postCreateProjectDetails = async (url, projectJson, taskAreaGeojson, formUpload) => { try { - const postNewProjectDetails = await axios.post(url, payload); + // Create project + const postNewProjectDetails = await axios.post(url, projectJson); const resp: ProjectDetailsModel = postNewProjectDetails.data; await dispatch(CreateProjectActions.PostProjectDetails(resp)); - if (payload.task_split_type === task_split_type['choose_area_as_task']) { - await dispatch( - UploadAreaService(`${import.meta.env.VITE_API_URL}/projects/${resp.id}/custom_task_boundaries`, fileUpload), - ); - } else if (payload.splitting_algorithm === 'Use natural Boundary') { - // TODO this is not longer valid, remove? - await dispatch( - UploadAreaService(`${import.meta.env.VITE_API_URL}/projects/task-split/${resp.id}/`, fileUpload), - ); - } else { - await dispatch( - UploadAreaService(`${import.meta.env.VITE_API_URL}/projects/${resp.id}/custom_task_boundaries`, fileUpload), - ); - // await dispatch(UploadAreaService(`${import.meta.env.VITE_API_URL}/projects/${resp.id}/upload`, fileUpload, { dimension: payload.dimension })); - } + // Submit task boundaries + await dispatch( + UploadTaskAreasService( + `${import.meta.env.VITE_API_URL}/projects/${resp.id}/upload-task-boundaries`, + taskAreaGeojson, + ), + ); + dispatch( CommonActions.SetSnackBar({ open: true, @@ -51,13 +45,16 @@ const CreateProjectService: Function = ( }), ); - if (payload.dataExtractWays === 'osm_data_extract') { + // FIXME not identifying osm_data_extract + console.log(projectJson.dataExtractWays); + if (projectJson.dataExtractWays === 'osm_data_extract') { + console.log('HERE'); // Upload data extract generated from raw-data-api const response = await axios.post( `${import.meta.env.VITE_API_URL}/projects/data-extract-url/?project_id=${resp.id}`, { - url: payload.data_extract_url, - extract_type: payload.data_extract_type, + url: projectJson.data_extract_url, + extract_type: projectJson.data_extract_type, }, ); } else if (dataExtractFile) { @@ -74,7 +71,7 @@ const CreateProjectService: Function = ( await dispatch( GenerateProjectQRService( `${import.meta.env.VITE_API_URL}/projects/${resp.id}/generate-project-data`, - payload, + projectJson, formUpload, ), ); @@ -101,7 +98,7 @@ const CreateProjectService: Function = ( } }; - await postCreateProjectDetails(url, payload, fileUpload, formUpload); + await postCreateProjectDetails(url, projectJson, taskAreaGeojson, formUpload); }; }; const FormCategoryService: Function = (url: string) => { @@ -121,16 +118,13 @@ const FormCategoryService: Function = (url: string) => { await getFormCategoryList(url); }; }; -const UploadAreaService: Function = (url: string, filePayload: any, payload: any) => { +const UploadTaskAreasService: Function = (url: string, filePayload: any, projectJson: any) => { return async (dispatch) => { dispatch(CreateProjectActions.UploadAreaLoading(true)); - const postUploadArea = async (url, filePayload, payload) => { + const postUploadArea = async (url, filePayload) => { try { const areaFormData = new FormData(); - areaFormData.append('project_geojson', filePayload); - if (payload?.dimension) { - areaFormData.append('dimension', payload?.dimension); - } + areaFormData.append('task_geojson', filePayload); const postNewProjectDetails = await axios.post(url, areaFormData, { headers: { 'Content-Type': 'multipart/form-data', @@ -153,19 +147,19 @@ const UploadAreaService: Function = (url: string, filePayload: any, payload: any } }; - await postUploadArea(url, filePayload, payload); + await postUploadArea(url, filePayload); }; }; -const GenerateProjectQRService: Function = (url: string, payload: any, formUpload: any) => { +const GenerateProjectQRService: Function = (url: string, projectJson: any, formUpload: any) => { return async (dispatch) => { dispatch(CreateProjectActions.GenerateProjectQRLoading(true)); dispatch(CommonActions.SetLoading(true)); - const postUploadArea = async (url, payload: any, formUpload) => { + const postUploadArea = async (url, projectJson: any, formUpload) => { try { let postNewProjectDetails; - if (payload.form_ways === 'custom_form') { + if (projectJson.form_ways === 'custom_form') { // TODO move form upload to a separate service / endpoint? const generateApiFormData = new FormData(); generateApiFormData.append('xls_form_upload', formUpload); @@ -197,7 +191,7 @@ const GenerateProjectQRService: Function = (url: string, payload: any, formUploa } }; - await postUploadArea(url, payload, formUpload); + await postUploadArea(url, projectJson, formUpload); }; }; @@ -236,15 +230,15 @@ const GenerateProjectLog: Function = (url: string, params: any) => { await getGenerateProjectLog(url, params); }; }; -const GetDividedTaskFromGeojson: Function = (url: string, payload: any) => { +const GetDividedTaskFromGeojson: Function = (url: string, projectJson: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetDividedTaskFromGeojsonLoading(true)); - const getDividedTaskFromGeojson = async (url, payload) => { + const getDividedTaskFromGeojson = async (url, projectJson) => { try { const dividedTaskFormData = new FormData(); - dividedTaskFormData.append('project_geojson', payload.geojson); - dividedTaskFormData.append('dimension', payload.dimension); + dividedTaskFormData.append('project_geojson', projectJson.geojson); + dividedTaskFormData.append('dimension', projectJson.dimension); const getGetDividedTaskFromGeojsonResponse = await axios.post(url, dividedTaskFormData); const resp: OrganisationListModel = getGetDividedTaskFromGeojsonResponse.data; dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'divide_on_square', value: true })); @@ -258,17 +252,17 @@ const GetDividedTaskFromGeojson: Function = (url: string, payload: any) => { } }; - await getDividedTaskFromGeojson(url, payload); + await getDividedTaskFromGeojson(url, projectJson); }; }; -const GetIndividualProjectDetails: Function = (url: string, payload: any) => { +const GetIndividualProjectDetails: Function = (url: string, projectJson: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetIndividualProjectDetailsLoading(true)); - const getIndividualProjectDetails = async (url, payload) => { + const getIndividualProjectDetails = async (url, projectJson) => { try { - const getIndividualProjectDetailsResponse = await axios.get(url, { params: payload }); + const getIndividualProjectDetailsResponse = await axios.get(url, { params: projectJson }); const resp: ProjectDetailsModel = getIndividualProjectDetailsResponse.data; const formattedOutlineGeojson = { type: 'FeatureCollection', features: [{ ...resp.outline_geojson, id: 1 }] }; const modifiedResponse = { @@ -288,24 +282,26 @@ const GetIndividualProjectDetails: Function = (url: string, payload: any) => { } }; - await getIndividualProjectDetails(url, payload); + await getIndividualProjectDetails(url, projectJson); }; }; const TaskSplittingPreviewService: Function = ( url: string, - fileUpload: any, + projectAoiFile: any, no_of_buildings: string, dataExtractFile: any, ) => { return async (dispatch) => { dispatch(CreateProjectActions.GetTaskSplittingPreviewLoading(true)); - const getTaskSplittingGeojson = async (url, fileUpload, dataExtractFile) => { + const getTaskSplittingGeojson = async (url, projectAoiFile, dataExtractFile) => { try { const taskSplittingFileFormData = new FormData(); - taskSplittingFileFormData.append('project_geojson', fileUpload); + taskSplittingFileFormData.append('project_geojson', projectAoiFile); taskSplittingFileFormData.append('no_of_buildings', no_of_buildings); + // Only include data extract if custom extract uploaded + console.log(dataExtractFile); if (dataExtractFile) { taskSplittingFileFormData.append('extract_geojson', dataExtractFile); } @@ -335,16 +331,16 @@ const TaskSplittingPreviewService: Function = ( } }; - await getTaskSplittingGeojson(url, fileUpload, dataExtractFile); + await getTaskSplittingGeojson(url, projectAoiFile, dataExtractFile); }; }; -const PatchProjectDetails: Function = (url: string, payload: any) => { +const PatchProjectDetails: Function = (url: string, projectJson: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetPatchProjectDetailsLoading(true)); - const patchProjectDetails = async (url, payload) => { + const patchProjectDetails = async (url, projectJson) => { try { - const getIndividualProjectDetailsResponse = await axios.patch(url, payload); + const getIndividualProjectDetailsResponse = await axios.patch(url, projectJson); const resp: ProjectDetailsModel = getIndividualProjectDetailsResponse.data; // dispatch(CreateProjectActions.SetIndividualProjectDetails(modifiedResponse)); dispatch(CreateProjectActions.SetPatchProjectDetails(resp)); @@ -364,22 +360,22 @@ const PatchProjectDetails: Function = (url: string, payload: any) => { } }; - await patchProjectDetails(url, payload); + await patchProjectDetails(url, projectJson); }; }; -const PostFormUpdate: Function = (url: string, payload: any) => { +const PostFormUpdate: Function = (url: string, projectJson: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetPostFormUpdateLoading(true)); - const postFormUpdate = async (url, payload) => { + const postFormUpdate = async (url, projectJson) => { try { const formFormData = new FormData(); - formFormData.append('project_id', payload.project_id); - if (payload.category) { - formFormData.append('category', payload.category); + formFormData.append('project_id', projectJson.project_id); + if (projectJson.category) { + formFormData.append('category', projectJson.category); } - if (payload.upload) { - formFormData.append('upload', payload.upload); + if (projectJson.upload) { + formFormData.append('upload', projectJson.upload); } const postFormUpdateResponse = await axios.post(url, formFormData); const resp: ProjectDetailsModel = postFormUpdateResponse.data; @@ -409,7 +405,7 @@ const PostFormUpdate: Function = (url: string, payload: any) => { } }; - await postFormUpdate(url, payload); + await postFormUpdate(url, projectJson); }; }; const EditProjectBoundaryService: Function = (url: string, geojsonUpload: any, dimension: any) => { @@ -529,7 +525,7 @@ const DeleteProjectService: Function = (url: string) => { }; export { - UploadAreaService, + UploadTaskAreasService, CreateProjectService, FormCategoryService, GenerateProjectQRService, diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index 97e5207ea9..3400930f63 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -89,12 +89,6 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo const submission = () => { dispatch(CreateProjectActions.SetIsUnsavedChanges(false)); - const projectAreaBlob = new Blob([JSON.stringify(dividedTaskGeojson || drawnGeojson)], { - type: 'application/json', - }); - // Create a file object from the Blob - const drawnGeojsonFile = new File([projectAreaBlob], 'data.json', { type: 'application/json' }); - dispatch(CreateProjectActions.SetIndividualProjectDetailsData(formValues)); const hashtags = projectDetails.hashtags; const arrayHashtag = hashtags @@ -102,12 +96,15 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo .map((item) => item.trim()) .filter(Boolean); + // Project POST data let projectData = { project_info: { name: projectDetails.name, short_description: projectDetails.short_description, description: projectDetails.description, }, + // Use split task areas, or project area if no task splitting + outline_geojson: dividedTaskGeojson || drawnGeojson, odk_central_url: projectDetails.odk_central_url, odk_central_user: projectDetails.odk_central_user, odk_central_password: projectDetails.odk_central_password, @@ -120,16 +117,24 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo data_extract_type: projectDetails.data_extract_type, data_extract_url: projectDetails.data_extract_url, }; + // Append extra param depending on task split type if (splitTasksSelection === task_split_type['task_splitting_algorithm']) { projectData = { ...projectData, task_num_buildings: projectDetails.average_buildings_per_task }; } else { projectData = { ...projectData, task_split_dimension: projectDetails.dimension }; } + // Create file object from generated task areas + const taskAreaBlob = new Blob([JSON.stringify(dividedTaskGeojson || drawnGeojson)], { + type: 'application/json', + }); + // Create a file object from the Blob + const taskAreaGeojsonFile = new File([taskAreaBlob], 'data.json', { type: 'application/json' }); + dispatch( CreateProjectService( `${import.meta.env.VITE_API_URL}/projects/create_project?org_id=${projectDetails.organisation_id}`, projectData, - drawnGeojsonFile, + taskAreaGeojsonFile, customFormFile, customPolygonUpload, customLineUpload, @@ -162,12 +167,12 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo const drawnGeojsonFile = new File([projectAreaBlob], 'outline.json', { type: 'application/json' }); // Create a file object from the data extract Blob - // const dataExtractBlob = new Blob([JSON.stringify(dataExtractGeojson)], { type: 'application/json' }); - // const dataExtractFile = new File([dataExtractBlob], 'extract.json', { type: 'application/json' }); + const dataExtractBlob = new Blob([JSON.stringify(dataExtractGeojson)], { type: 'application/json' }); + const dataExtractFile = new File([dataExtractBlob], 'extract.json', { type: 'application/json' }); if (splitTasksSelection === task_split_type['divide_on_square']) { dispatch( - GetDividedTaskFromGeojson(`${import.meta.env.VITE_API_URL}/projects/preview_split_by_square/`, { + GetDividedTaskFromGeojson(`${import.meta.env.VITE_API_URL}/projects/preview-split-by-square/`, { geojson: drawnGeojsonFile, dimension: formValues?.dimension, }), @@ -182,8 +187,8 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo `${import.meta.env.VITE_API_URL}/projects/task-split`, drawnGeojsonFile, formValues?.average_buildings_per_task, - // TODO include extract file only if custom upload - // dataExtractFile, + // Only send dataExtractFile if custom extract + formValues.dataExtractWays === 'osm_data_extract' ? null : dataExtractFile, ), ); } From 3453fe16d09bf5996be9328a77e6331d41eef299 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 16:54:17 +0000 Subject: [PATCH 5/9] fix: optional project_task_name in schema --- src/backend/app/projects/project_crud.py | 6 ++---- src/backend/app/projects/project_routes.py | 2 +- src/backend/app/tasks/tasks_schemas.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index dfb443c500..3217d5127b 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -593,17 +593,15 @@ def remove_z_dimension(coord): boundary, meters=meters, ) - for poly in tasks["features"]: + for index, poly in enumerate(tasks["features"]): log.debug(poly) - task_id = str(poly.get("properties", {}).get("id") or poly.get("id")) db_task = db_models.DbTask( project_id=project_id, - project_task_name=task_id, outline=wkblib.dumps(shape(poly["geometry"]), hex=True), # qr_code=db_qr, # qr_code_id=db_qr.id, # project_task_index=feature["properties"]["fid"], - project_task_index=1, + project_task_index=index, # geometry_geojson=geojson.dumps(task_geojson), # initial_feature_count=len(task_geojson["features"]), ) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 8e4ce631c0..076ab9eab6 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -94,7 +94,7 @@ async def get_projet_details( """ project = await project_crud.get_project(db, project_id) if not project: - raise HTTPException(status_code=404, details={"Project not found"}) + raise HTTPException(status_code=404, detail={"Project not found"}) # ODK Credentials odk_credentials = project_schemas.ODKCentralDecrypted( diff --git a/src/backend/app/tasks/tasks_schemas.py b/src/backend/app/tasks/tasks_schemas.py index 175a9ebded..96ea8fe213 100644 --- a/src/backend/app/tasks/tasks_schemas.py +++ b/src/backend/app/tasks/tasks_schemas.py @@ -70,7 +70,7 @@ class Task(BaseModel): id: int project_id: int project_task_index: int - project_task_name: str + project_task_name: Optional[str] outline_geojson: Optional[GeojsonFeature] = None outline_centroid: Optional[GeojsonFeature] = None initial_feature_count: Optional[int] = None From 4ed3e81e07fff101adbd7d472cab8b3e130c2516 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 16:54:43 +0000 Subject: [PATCH 6/9] refactor(frontend): remove refs to project_task_name --- src/frontend/src/api/CreateProjectService.ts | 1 - src/frontend/src/api/Project.js | 1 - src/frontend/src/models/createproject/createProjectModel.ts | 1 - src/frontend/src/store/types/ICreateProject.ts | 1 - src/frontend/src/views/NewProjectDetails.jsx | 2 +- src/frontend/src/views/ProjectDetailsV2.tsx | 2 +- 6 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index a2826403b2..ce2a5fdfbb 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -301,7 +301,6 @@ const TaskSplittingPreviewService: Function = ( taskSplittingFileFormData.append('project_geojson', projectAoiFile); taskSplittingFileFormData.append('no_of_buildings', no_of_buildings); // Only include data extract if custom extract uploaded - console.log(dataExtractFile); if (dataExtractFile) { taskSplittingFileFormData.append('extract_geojson', dataExtractFile); } diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 1755b3f2b7..60c749eac7 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -20,7 +20,6 @@ export const ProjectById = (existingProjectList, projectId) => { const persistingValues = taskListResp.map((data) => { return { id: data.id, - project_task_name: data.project_task_name, outline_geojson: data.outline_geojson, outline_centroid: data.outline_centroid, task_status: task_priority_str[data.task_status], diff --git a/src/frontend/src/models/createproject/createProjectModel.ts b/src/frontend/src/models/createproject/createProjectModel.ts index 11c0d5f6d2..58bdd6de3a 100755 --- a/src/frontend/src/models/createproject/createProjectModel.ts +++ b/src/frontend/src/models/createproject/createProjectModel.ts @@ -27,7 +27,6 @@ export interface ProjectDetailsModel { id: number; project_id: number; project_task_index: number; - project_task_name: string; outline_geojson: { type: string; geometry: { diff --git a/src/frontend/src/store/types/ICreateProject.ts b/src/frontend/src/store/types/ICreateProject.ts index 17f769bd9a..94ba54b922 100644 --- a/src/frontend/src/store/types/ICreateProject.ts +++ b/src/frontend/src/store/types/ICreateProject.ts @@ -59,7 +59,6 @@ export type ProjectTaskTypes = { id: number; project_id: number; project_task_index: number; - project_task_name: string; outline_geojson: GeoJSONFeatureTypes; outline_centroid: GeoJSONFeatureTypes; task_status: number; diff --git a/src/frontend/src/views/NewProjectDetails.jsx b/src/frontend/src/views/NewProjectDetails.jsx index 6249b6cbbf..cffb47b8b6 100644 --- a/src/frontend/src/views/NewProjectDetails.jsx +++ b/src/frontend/src/views/NewProjectDetails.jsx @@ -121,7 +121,7 @@ const Home = () => { ...feature.outline_geojson.properties, centroid: feature.bbox, }, - id: `${feature.project_task_name}_${feature.task_status}`, + id: `${feature.id}_${feature.task_status}`, })); const taskBuildingGeojsonFeatureCollection = { ...geojsonObjectModel, diff --git a/src/frontend/src/views/ProjectDetailsV2.tsx b/src/frontend/src/views/ProjectDetailsV2.tsx index f634271a05..7739474930 100644 --- a/src/frontend/src/views/ProjectDetailsV2.tsx +++ b/src/frontend/src/views/ProjectDetailsV2.tsx @@ -125,7 +125,7 @@ const Home = () => { ...feature.outline_geojson.properties, centroid: feature.bbox, }, - id: `${feature.project_task_name}_${feature.task_status}`, + id: `${feature.id}_${feature.task_status}`, })); const taskBuildingGeojsonFeatureCollection = { ...geojsonObjectModel, From b0d57daac068d0d87200bcee3c8003529c6625dd Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 17:16:41 +0000 Subject: [PATCH 7/9] refactor(backend): remove extract_type from data-extract-url --- src/backend/app/projects/project_crud.py | 10 +++------- src/backend/app/projects/project_routes.py | 4 +--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 3217d5127b..da72d8c6d4 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -881,7 +881,6 @@ async def get_or_set_data_extract_url( db: Session, project_id: int, url: Optional[str], - extract_type: Optional[str], ) -> str: """Get or set the data extract URL for a project. @@ -890,8 +889,6 @@ async def get_or_set_data_extract_url( project_id (int): The ID of the project. url (str): URL to the streamable flatgeobuf data extract. If not passed, a new extract is generated. - extract_type (str): The type of data extract, required if setting URL - in database. Returns: str: URL to fgb file in S3. @@ -916,10 +913,9 @@ async def get_or_set_data_extract_url( raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) return existing_url - if not extract_type: - msg = "The extract_type param is required if URL is set." - log.error(msg) - raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) + # FIXME Identify data extract type from form type + # FIXME use mapping e.g. building=polygon, waterways=line, etc + extract_type = "polygon" await update_data_extract_url_in_db(db, db_project, url, extract_type) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 076ab9eab6..8ab59d61be 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -843,10 +843,9 @@ async def get_data_extract( return JSONResponse(status_code=200, content={"url": fgb_url}) -@router.post("/data-extract-url/") +@router.get("/data-extract-url/") async def get_or_set_data_extract( url: Optional[str] = None, - extract_type: Optional[str] = None, project_id: int = Query(..., description="Project ID"), db: Session = Depends(database.get_db), org_user_dict: db_models.DbUser = Depends(project_admin), @@ -856,7 +855,6 @@ async def get_or_set_data_extract( db, project_id, url, - extract_type, ) return JSONResponse(status_code=200, content={"url": fgb_url}) From 3d4f23373f7d98eef76ee94a4c470eb714b0d0f2 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 17:17:16 +0000 Subject: [PATCH 8/9] fix(frontend): setting data extract url if osm generated extract --- src/frontend/src/api/CreateProjectService.ts | 78 +++++++++---------- .../createnewproject/SplitTasks.tsx | 4 +- 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index ce2a5fdfbb..4de671c7cb 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -12,19 +12,20 @@ import { task_split_type } from '@/types/enums'; const CreateProjectService: Function = ( url: string, - projectJson: any, + projectData: any, taskAreaGeojson: any, formUpload: any, dataExtractFile: any, + isOsmExtract: boolean, ) => { return async (dispatch) => { dispatch(CreateProjectActions.CreateProjectLoading(true)); dispatch(CommonActions.SetLoading(true)); - const postCreateProjectDetails = async (url, projectJson, taskAreaGeojson, formUpload) => { + const postCreateProjectDetails = async (url, projectData, taskAreaGeojson, formUpload) => { try { // Create project - const postNewProjectDetails = await axios.post(url, projectJson); + const postNewProjectDetails = await axios.post(url, projectData); const resp: ProjectDetailsModel = postNewProjectDetails.data; await dispatch(CreateProjectActions.PostProjectDetails(resp)); @@ -45,17 +46,12 @@ const CreateProjectService: Function = ( }), ); - // FIXME not identifying osm_data_extract - console.log(projectJson.dataExtractWays); - if (projectJson.dataExtractWays === 'osm_data_extract') { - console.log('HERE'); + if (isOsmExtract) { // Upload data extract generated from raw-data-api - const response = await axios.post( - `${import.meta.env.VITE_API_URL}/projects/data-extract-url/?project_id=${resp.id}`, - { - url: projectJson.data_extract_url, - extract_type: projectJson.data_extract_type, - }, + const response = await axios.get( + `${import.meta.env.VITE_API_URL}/projects/data-extract-url/?project_id=${resp.id}&url=${ + projectData.data_extract_url + }`, ); } else if (dataExtractFile) { // Upload custom data extract from user @@ -71,7 +67,7 @@ const CreateProjectService: Function = ( await dispatch( GenerateProjectQRService( `${import.meta.env.VITE_API_URL}/projects/${resp.id}/generate-project-data`, - projectJson, + projectData, formUpload, ), ); @@ -98,7 +94,7 @@ const CreateProjectService: Function = ( } }; - await postCreateProjectDetails(url, projectJson, taskAreaGeojson, formUpload); + await postCreateProjectDetails(url, projectData, taskAreaGeojson, formUpload); }; }; const FormCategoryService: Function = (url: string) => { @@ -118,7 +114,7 @@ const FormCategoryService: Function = (url: string) => { await getFormCategoryList(url); }; }; -const UploadTaskAreasService: Function = (url: string, filePayload: any, projectJson: any) => { +const UploadTaskAreasService: Function = (url: string, filePayload: any, projectData: any) => { return async (dispatch) => { dispatch(CreateProjectActions.UploadAreaLoading(true)); const postUploadArea = async (url, filePayload) => { @@ -150,16 +146,16 @@ const UploadTaskAreasService: Function = (url: string, filePayload: any, project await postUploadArea(url, filePayload); }; }; -const GenerateProjectQRService: Function = (url: string, projectJson: any, formUpload: any) => { +const GenerateProjectQRService: Function = (url: string, projectData: any, formUpload: any) => { return async (dispatch) => { dispatch(CreateProjectActions.GenerateProjectQRLoading(true)); dispatch(CommonActions.SetLoading(true)); - const postUploadArea = async (url, projectJson: any, formUpload) => { + const postUploadArea = async (url, projectData: any, formUpload) => { try { let postNewProjectDetails; - if (projectJson.form_ways === 'custom_form') { + if (projectData.form_ways === 'custom_form') { // TODO move form upload to a separate service / endpoint? const generateApiFormData = new FormData(); generateApiFormData.append('xls_form_upload', formUpload); @@ -191,7 +187,7 @@ const GenerateProjectQRService: Function = (url: string, projectJson: any, formU } }; - await postUploadArea(url, projectJson, formUpload); + await postUploadArea(url, projectData, formUpload); }; }; @@ -230,15 +226,15 @@ const GenerateProjectLog: Function = (url: string, params: any) => { await getGenerateProjectLog(url, params); }; }; -const GetDividedTaskFromGeojson: Function = (url: string, projectJson: any) => { +const GetDividedTaskFromGeojson: Function = (url: string, projectData: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetDividedTaskFromGeojsonLoading(true)); - const getDividedTaskFromGeojson = async (url, projectJson) => { + const getDividedTaskFromGeojson = async (url, projectData) => { try { const dividedTaskFormData = new FormData(); - dividedTaskFormData.append('project_geojson', projectJson.geojson); - dividedTaskFormData.append('dimension', projectJson.dimension); + dividedTaskFormData.append('project_geojson', projectData.geojson); + dividedTaskFormData.append('dimension', projectData.dimension); const getGetDividedTaskFromGeojsonResponse = await axios.post(url, dividedTaskFormData); const resp: OrganisationListModel = getGetDividedTaskFromGeojsonResponse.data; dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'divide_on_square', value: true })); @@ -252,17 +248,17 @@ const GetDividedTaskFromGeojson: Function = (url: string, projectJson: any) => { } }; - await getDividedTaskFromGeojson(url, projectJson); + await getDividedTaskFromGeojson(url, projectData); }; }; -const GetIndividualProjectDetails: Function = (url: string, projectJson: any) => { +const GetIndividualProjectDetails: Function = (url: string, projectData: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetIndividualProjectDetailsLoading(true)); - const getIndividualProjectDetails = async (url, projectJson) => { + const getIndividualProjectDetails = async (url, projectData) => { try { - const getIndividualProjectDetailsResponse = await axios.get(url, { params: projectJson }); + const getIndividualProjectDetailsResponse = await axios.get(url, { params: projectData }); const resp: ProjectDetailsModel = getIndividualProjectDetailsResponse.data; const formattedOutlineGeojson = { type: 'FeatureCollection', features: [{ ...resp.outline_geojson, id: 1 }] }; const modifiedResponse = { @@ -282,7 +278,7 @@ const GetIndividualProjectDetails: Function = (url: string, projectJson: any) => } }; - await getIndividualProjectDetails(url, projectJson); + await getIndividualProjectDetails(url, projectData); }; }; @@ -333,13 +329,13 @@ const TaskSplittingPreviewService: Function = ( await getTaskSplittingGeojson(url, projectAoiFile, dataExtractFile); }; }; -const PatchProjectDetails: Function = (url: string, projectJson: any) => { +const PatchProjectDetails: Function = (url: string, projectData: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetPatchProjectDetailsLoading(true)); - const patchProjectDetails = async (url, projectJson) => { + const patchProjectDetails = async (url, projectData) => { try { - const getIndividualProjectDetailsResponse = await axios.patch(url, projectJson); + const getIndividualProjectDetailsResponse = await axios.patch(url, projectData); const resp: ProjectDetailsModel = getIndividualProjectDetailsResponse.data; // dispatch(CreateProjectActions.SetIndividualProjectDetails(modifiedResponse)); dispatch(CreateProjectActions.SetPatchProjectDetails(resp)); @@ -359,22 +355,22 @@ const PatchProjectDetails: Function = (url: string, projectJson: any) => { } }; - await patchProjectDetails(url, projectJson); + await patchProjectDetails(url, projectData); }; }; -const PostFormUpdate: Function = (url: string, projectJson: any) => { +const PostFormUpdate: Function = (url: string, projectData: any) => { return async (dispatch) => { dispatch(CreateProjectActions.SetPostFormUpdateLoading(true)); - const postFormUpdate = async (url, projectJson) => { + const postFormUpdate = async (url, projectData) => { try { const formFormData = new FormData(); - formFormData.append('project_id', projectJson.project_id); - if (projectJson.category) { - formFormData.append('category', projectJson.category); + formFormData.append('project_id', projectData.project_id); + if (projectData.category) { + formFormData.append('category', projectData.category); } - if (projectJson.upload) { - formFormData.append('upload', projectJson.upload); + if (projectData.upload) { + formFormData.append('upload', projectData.upload); } const postFormUpdateResponse = await axios.post(url, formFormData); const resp: ProjectDetailsModel = postFormUpdateResponse.data; @@ -404,7 +400,7 @@ const PostFormUpdate: Function = (url: string, projectJson: any) => { } }; - await postFormUpdate(url, projectJson); + await postFormUpdate(url, projectData); }; }; const EditProjectBoundaryService: Function = (url: string, geojsonUpload: any, dimension: any) => { diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index 3400930f63..4d9c0781e8 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -136,8 +136,8 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo projectData, taskAreaGeojsonFile, customFormFile, - customPolygonUpload, - customLineUpload, + customPolygonUpload || customLineUpload, + projectDetails.dataExtractWays === 'osm_data_extract', ), ); dispatch(CreateProjectActions.SetIndividualProjectDetailsData({ ...projectDetails, ...formValues })); From 9615d99a2d6cc07ab70ddff1478569e736a5f21f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 12 Feb 2024 19:09:54 +0000 Subject: [PATCH 9/9] test: fix or remove tests during project creation --- src/backend/app/projects/project_crud.py | 2 +- src/backend/tests/conftest.py | 16 ++++ src/backend/tests/test_projects_routes.py | 104 +++++++++++++--------- 3 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index da72d8c6d4..0866bc9cc3 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -528,7 +528,7 @@ async def update_project_boundary( ): """Update the boundary for a project and update tasks. - TODO this needs a big refactor / removal + TODO this needs a big refactor or removal # """ # verify project exists in db diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index 46f43efdbe..6db87f86e4 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -134,6 +134,22 @@ async def project(db, admin_user, organisation): odk_central_password=os.getenv("ODK_CENTRAL_PASSWD"), hashtags=["hot-fmtm"], organisation_id=organisation.id, + outline_geojson={ + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [85.317028828, 27.7052522097], + [85.317028828, 27.7041424888], + [85.318844411, 27.7041424888], + [85.318844411, 27.7052522097], + [85.317028828, 27.7052522097], + ] + ], + "type": "Polygon", + }, + }, ) odk_creds_decrypted = ODKCentralDecrypted( diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index 870a87ae4a..6e9fae24ec 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -18,7 +18,6 @@ """Tests for project routes.""" import functools -import json import os import uuid from io import BytesIO @@ -30,8 +29,7 @@ from fastapi.concurrency import run_in_threadpool from geoalchemy2.elements import WKBElement from loguru import logger as log -from shapely import Polygon, wkb -from shapely.geometry import shape +from shapely import Polygon from app.central.central_crud import create_odk_project from app.config import encrypt_value, settings @@ -62,6 +60,18 @@ async def test_create_project(client, admin_user, organisation): }, "xform_title": "buildings", "hashtags": ["#FMTM"], + "outline_geojson": { + "coordinates": [ + [ + [85.317028828, 27.7052522097], + [85.317028828, 27.7041424888], + [85.318844411, 27.7041424888], + [85.318844411, 27.7052522097], + [85.317028828, 27.7052522097], + ] + ], + "type": "Polygon", + }, } project_data.update(**odk_credentials.model_dump()) @@ -97,12 +107,11 @@ async def test_convert_to_app_project(): """Test conversion ot app project.""" polygon = Polygon( [ - (85.924707758, 26.727727503), - (85.922703741, 26.732440043), - (85.928284549, 26.735158727), - (85.930643709, 26.734365785), - (85.932368686, 26.732372075), - (85.924707758, 26.727727503), + (85.317028828, 27.7052522097), + (85.317028828, 27.7041424888), + (85.318844411, 27.7041424888), + (85.318844411, 27.7052522097), + (85.317028828, 27.7052522097), ] ) @@ -142,41 +151,6 @@ async def test_generate_appuser_files(db, project): project_id = project.id log.debug(f"Testing project ID: {project_id}") - # Set project boundary - boundary_geojson = json.loads( - json.dumps( - { - "type": "Polygon", - "coordinates": [ - [ - [8.539551723844, 47.3765788922656], - [8.539551723844, 47.37303247378486], - [8.547135285454686, 47.37303247378486], - [8.547135285454686, 47.3765788922656], - [8.539551723844, 47.3765788922656], - ] - ], - } - ) - ) - log.debug(f"Creating project boundary: {boundary_geojson}") - boundary_created = await project_crud.update_project_boundary( - db, project_id, boundary_geojson, 500 - ) - assert boundary_created is True - # Check updated locations - db_project = await project_crud.get_project_by_id(db, project_id) - # Outline - project_outline = db_project.outline.data.tobytes() - file_outline = shape(boundary_geojson) - assert wkb.loads(project_outline).wkt == file_outline.wkt - # Centroid - project_centroid = wkb.loads(db_project.centroid.data.tobytes()).wkt - file_centroid = file_outline.centroid.wkt - assert project_centroid == file_centroid - # Location string - assert db_project.location_str == "Zurich,Switzerland" - # Load data extracts data_extracts_file = f"{test_data_path}/building_footprint.zip" with zipfile.ZipFile(data_extracts_file, "r") as zip_archive: @@ -239,6 +213,48 @@ async def test_generate_appuser_files(db, project): assert result is None +# async def test_update_project_boundary(db, project): +# """Test updating project boundary.""" +# project_id = project.id +# log.debug(f"Testing updating boundary for project ID: {project_id}") + +# db_project = await project_crud.get_project_by_id(db, project_id) + +# # Outline +# boundary_geojson = json.loads( +# json.dumps( +# { +# "type": "Polygon", +# "coordinates": [ +# [ +# [85.317028828, 27.7052522097], +# [85.317028828, 27.7041424888], +# [85.318844411, 27.7041424888], +# [85.318844411, 27.7052522097], +# [85.317028828, 27.7052522097], +# ] +# ], +# } +# ) +# ) +# log.debug(f"Creating project boundary: {boundary_geojson}") +# boundary_created = await project_crud.update_project_boundary( +# db, project_id, boundary_geojson, 500 +# ) +# assert boundary_created is True +# project_outline = db_project.outline.data.tobytes() +# file_outline = shape(boundary_geojson) +# assert wkb.loads(project_outline).wkt == file_outline.wkt + +# # Centroid +# project_centroid = wkb.loads(db_project.centroid.data.tobytes()).wkt +# file_centroid = file_outline.centroid.wkt +# assert project_centroid == file_centroid + +# # Location string +# assert db_project.location_str == "Zurich,Switzerland" + + if __name__ == "__main__": """Main func if file invoked directly.""" pytest.main()