From 9654218aff9055e4f6bf2d66ca6fd13120507d5c Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:02:45 +0000 Subject: [PATCH] fix: map default odk credentials to organisations (#1123) * fix: add odk cred org schemas for db and pydantic * fix: add HttpUrlStr type for Url validation of str values * fix: fix OdkCentral schema to handle passwords, inherit * refactor: use new ODKCentralDecrypted, simplify proj create * fix: add org_deps get_org_odk_creds method * refactor(frontend): update create_project json structure * refactor(frontend): remove code to add #FMTM tag automatically * test: fix project route tests * refactor: revert setting outline_geojson in create_app_project * feat(frontend): option to add default odk creds for organisation * refactor(frontend): rename form during project creation --> category * refactor: remove additional route added during rebase --- src/backend/app/central/central_crud.py | 44 +++--- src/backend/app/central/central_routes.py | 2 +- src/backend/app/config.py | 15 +- .../app/organisations/organisation_crud.py | 2 +- .../app/organisations/organisation_deps.py | 57 +++++-- .../app/organisations/organisation_routes.py | 2 +- .../app/organisations/organisation_schemas.py | 57 +------ src/backend/app/projects/project_crud.py | 110 ++++---------- src/backend/app/projects/project_routes.py | 43 +++++- src/backend/app/projects/project_schemas.py | 139 ++++++++++++++---- .../app/submissions/submission_crud.py | 20 +-- src/backend/app/tasks/tasks_routes.py | 2 +- src/backend/app/tasks/tasks_schemas.py | 6 +- src/backend/tests/conftest.py | 34 +++-- src/backend/tests/test_projects_routes.py | 24 +-- .../LayerSwitcher/index.js | 2 +- .../createnewproject/DataExtract.tsx | 4 +- .../createnewproject/ProjectDetailsForm.tsx | 18 --- .../createnewproject/SelectForm.tsx | 6 +- .../createnewproject/SplitTasks.tsx | 13 +- .../createnewproject/UploadArea.tsx | 10 +- .../organisation/OrganisationAddForm.tsx | 1 - .../Validation/OrganisationAddValidation.tsx | 19 ++- .../src/constants/StepFormConstants.ts | 4 +- src/frontend/src/constants/blockerUrl.ts | 2 +- .../createproject/createProjectModel.ts | 4 - .../models/organisation/organisationModel.ts | 9 -- src/frontend/src/routes.jsx | 4 +- .../src/store/slices/CreateProjectSlice.ts | 4 +- .../src/store/types/ICreateProject.ts | 5 - src/frontend/src/views/CreateNewProject.tsx | 4 +- src/frontend/src/views/CreateOrganisation.tsx | 63 ++++++++ src/frontend/src/views/CreateProject.tsx | 12 +- 33 files changed, 413 insertions(+), 328 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 185f557760..1caccc4df2 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -35,12 +35,12 @@ from app.projects import project_schemas -def get_odk_project(odk_central: project_schemas.ODKCentral = None): +def get_odk_project(odk_central: project_schemas.ODKCentralDecrypted = None): """Helper function to get the OdkProject with credentials.""" if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password.get_secret_value() + pw = odk_central.odk_central_password else: log.debug("ODKCentral connection variables not set in function") log.debug("Attempting extraction from environment variables") @@ -60,12 +60,12 @@ def get_odk_project(odk_central: project_schemas.ODKCentral = None): return project -def get_odk_form(odk_central: project_schemas.ODKCentral = None): +def get_odk_form(odk_central: project_schemas.ODKCentralDecrypted = None): """Helper function to get the OdkForm with credentials.""" if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password.get_secret_value() + pw = odk_central.odk_central_password else: log.debug("ODKCentral connection variables not set in function") @@ -86,12 +86,12 @@ def get_odk_form(odk_central: project_schemas.ODKCentral = None): return form -def get_odk_app_user(odk_central: project_schemas.ODKCentral = None): +def get_odk_app_user(odk_central: project_schemas.ODKCentralDecrypted = None): """Helper function to get the OdkAppUser with credentials.""" if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password.get_secret_value() + pw = odk_central.odk_central_password else: log.debug("ODKCentral connection variables not set in function") log.debug("Attempting extraction from environment variables") @@ -111,13 +111,15 @@ def get_odk_app_user(odk_central: project_schemas.ODKCentral = None): return form -def list_odk_projects(odk_central: project_schemas.ODKCentral = None): +def list_odk_projects(odk_central: project_schemas.ODKCentralDecrypted = None): """List all projects on a remote ODK Server.""" project = get_odk_project(odk_central) return project.listProjects() -def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None): +def create_odk_project( + name: str, odk_central: project_schemas.ODKCentralDecrypted = None +): """Create a project on a remote ODK Server.""" project = get_odk_project(odk_central) @@ -144,7 +146,7 @@ def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None async def delete_odk_project( - project_id: int, odk_central: project_schemas.ODKCentral = None + project_id: int, odk_central: project_schemas.ODKCentralDecrypted = None ): """Delete a project from a remote ODK Server.""" # FIXME: when a project is deleted from Central, we have to update the @@ -159,7 +161,7 @@ async def delete_odk_project( def delete_odk_app_user( - project_id: int, name: str, odk_central: project_schemas.ODKCentral = None + project_id: int, name: str, odk_central: project_schemas.ODKCentralDecrypted = None ): """Delete an app-user from a remote ODK Server.""" odk_app_user = get_odk_app_user(odk_central) @@ -202,7 +204,7 @@ def create_odk_xform( project_id: int, xform_id: str, filespec: str, - odk_credentials: project_schemas.ODKCentral = None, + odk_credentials: project_schemas.ODKCentralDecrypted = None, create_draft: bool = False, upload_media=True, convert_to_draft_when_publishing=True, @@ -213,7 +215,7 @@ def create_odk_xform( # Pass odk credentials of project in xform if not odk_credentials: - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=settings.ODK_CENTRAL_URL, odk_central_user=settings.ODK_CENTRAL_USER, odk_central_password=settings.ODK_CENTRAL_PASSWD, @@ -245,7 +247,7 @@ def delete_odk_xform( project_id: int, xform_id: str, filespec: str, - odk_central: project_schemas.ODKCentral = None, + odk_central: project_schemas.ODKCentralDecrypted = None, ): """Delete an XForm from a remote ODK Central server.""" xform = get_odk_form(odk_central) @@ -256,7 +258,7 @@ def delete_odk_xform( def list_odk_xforms( project_id: int, - odk_central: project_schemas.ODKCentral = None, + odk_central: project_schemas.ODKCentralDecrypted = None, metadata: bool = False, ): """List all XForms in an ODK Central project.""" @@ -267,7 +269,7 @@ def list_odk_xforms( def get_form_full_details( - odk_project_id: int, form_id: str, odk_central: project_schemas.ODKCentral + odk_project_id: int, form_id: str, odk_central: project_schemas.ODKCentralDecrypted ): """Get additional metadata for ODK Form.""" form = get_odk_form(odk_central) @@ -276,7 +278,7 @@ def get_form_full_details( def get_odk_project_full_details( - odk_project_id: int, odk_central: project_schemas.ODKCentral + odk_project_id: int, odk_central: project_schemas.ODKCentralDecrypted ): """Get additional metadata for ODK project.""" project = get_odk_project(odk_central) @@ -284,7 +286,9 @@ def get_odk_project_full_details( return project_details -def list_submissions(project_id: int, odk_central: project_schemas.ODKCentral = None): +def list_submissions( + project_id: int, odk_central: project_schemas.ODKCentralDecrypted = None +): """List all submissions for a project, aggregated from associated users.""" project = get_odk_project(odk_central) xform = get_odk_form(odk_central) @@ -326,7 +330,7 @@ def download_submissions( xform_id: str, submission_id: str = None, get_json: bool = True, - odk_central: project_schemas.ODKCentral = None, + odk_central: project_schemas.ODKCentralDecrypted = None, ): """Download all submissions for an XForm.""" xform = get_odk_form(odk_central) @@ -506,7 +510,7 @@ def upload_media( project_id: int, xform_id: str, filespec: str, - odk_central: project_schemas.ODKCentral = None, + odk_central: project_schemas.ODKCentralDecrypted = None, ): """Upload a data file to Central.""" xform = get_odk_form(odk_central) @@ -517,7 +521,7 @@ def download_media( project_id: int, xform_id: str, filespec: str, - odk_central: project_schemas.ODKCentral = None, + odk_central: project_schemas.ODKCentralDecrypted = None, ): """Upload a data file to Central.""" xform = get_odk_form(odk_central) diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 60e7f13997..4bd51a5ff7 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -211,7 +211,7 @@ async def get_submission( return {"error": "No such project!"} # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=first.odk_central_url, odk_central_user=first.odk_central_user, odk_central_password=first.odk_central_password, diff --git a/src/backend/app/config.py b/src/backend/app/config.py index cd422d964f..1ce27408bf 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -19,12 +19,17 @@ import base64 from functools import lru_cache -from typing import Any, Optional, Union +from typing import Annotated, Any, Optional, Union from cryptography.fernet import Fernet -from pydantic import PostgresDsn, ValidationInfo, field_validator +from pydantic import BeforeValidator, TypeAdapter, ValidationInfo, field_validator +from pydantic.networks import HttpUrl, PostgresDsn from pydantic_settings import BaseSettings, SettingsConfigDict +HttpUrlStr = Annotated[ + str, BeforeValidator(lambda value: str(TypeAdapter(HttpUrl).validate_python(value))) +] + class Settings(BaseSettings): """Main settings class, defining environment variables.""" @@ -100,14 +105,14 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: ) return pg_url - ODK_CENTRAL_URL: Optional[str] = "" + ODK_CENTRAL_URL: Optional[HttpUrlStr] = "" ODK_CENTRAL_USER: Optional[str] = "" ODK_CENTRAL_PASSWD: Optional[str] = "" OSM_CLIENT_ID: str OSM_CLIENT_SECRET: str OSM_SECRET_KEY: str - OSM_URL: str = "https://www.openstreetmap.org" + OSM_URL: HttpUrlStr = "https://www.openstreetmap.org" OSM_SCOPE: str = "read_prefs" OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:7051/osmauth/" @@ -147,7 +152,7 @@ def configure_s3_download_root(cls, v: Optional[str], info: ValidationInfo) -> s return f"http://s3.{fmtm_domain}:{dev_port}" return f"https://s3.{fmtm_domain}" - UNDERPASS_API_URL: str = "https://api-prod.raw-data.hotosm.org/v1/" + UNDERPASS_API_URL: HttpUrlStr = "https://api-prod.raw-data.hotosm.org/v1/" SENTRY_DSN: Optional[str] = None model_config = SettingsConfigDict( diff --git a/src/backend/app/organisations/organisation_crud.py b/src/backend/app/organisations/organisation_crud.py index 36e6ca649d..de3371780a 100644 --- a/src/backend/app/organisations/organisation_crud.py +++ b/src/backend/app/organisations/organisation_crud.py @@ -98,7 +98,7 @@ async def create_organisation( if await get_organisation_by_name(db, org_name=org_model.name): raise HTTPException( status_code=HTTPStatus.CONFLICT, - detail=f"Organisation already exists with the name {org_model.name}", + detail=f"Organisation already exists with the name ({org_model.name})", ) # Required to check if exists on error diff --git a/src/backend/app/organisations/organisation_deps.py b/src/backend/app/organisations/organisation_deps.py index 36c147302d..f2d18a1fde 100644 --- a/src/backend/app/organisations/organisation_deps.py +++ b/src/backend/app/organisations/organisation_deps.py @@ -23,13 +23,12 @@ from fastapi import Depends from fastapi.exceptions import HTTPException from loguru import logger as log -from sqlalchemy import func from sqlalchemy.orm import Session from app.db.database import get_db from app.db.db_models import DbOrganisation, DbProject from app.models.enums import HTTPStatus -from app.projects import project_deps +from app.projects import project_deps, project_schemas async def get_organisation_by_name( @@ -45,16 +44,20 @@ async def get_organisation_by_name( Returns: DbOrganisation: organisation with the given id """ - org_obj = ( - db.query(DbOrganisation) - .filter(func.lower(DbOrganisation.name).like(func.lower(f"%{org_name}%"))) - .first() - ) + # # For getting org with LIKE match + # org_obj = ( + # db.query(DbOrganisation) + # .filter(func.lower(DbOrganisation.name).like(func.lower(f"%{org_name}%"))) + # .first() + # ) + org_obj = db.query(DbOrganisation).filter_by(name=org_name).first() + if org_obj and check_approved and org_obj.approved is False: raise HTTPException( - status_code=HTTPStatus.NOT_FOUND, + status_code=HTTPStatus.FORBIDDEN, detail=f"Organisation ({org_obj.id}) is not approved yet", ) + return org_obj @@ -72,23 +75,51 @@ async def get_organisation_by_id( DbOrganisation: organisation with the given id """ org_obj = db.query(DbOrganisation).filter_by(id=org_id).first() + if org_obj and check_approved and org_obj.approved is False: raise HTTPException( - status_code=HTTPStatus.NOT_FOUND, - detail=f"Organisation {org_id} is not approved yet", + status_code=HTTPStatus.FORBIDDEN, + detail=f"Organisation ({org_id}) is not approved yet", ) return org_obj +async def get_org_odk_creds( + org: DbOrganisation, +) -> project_schemas.ODKCentralDecrypted: + """Get odk credentials for an organisation, else error.""" + url = org.odk_central_url + user = org.odk_central_user + password = org.odk_central_password + + if not all([url, user, password]): + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail="Organisation does not have ODK Central credentials configured", + ) + + return project_schemas.ODKCentralDecrypted( + odk_central_url=org.odk_central_url, + odk_central_user=org.odk_central_user, + odk_central_password=org.odk_central_password, + ) + + async def check_org_exists( db: Session, - org_id: Union[str, int], + org_id: Union[str, int, None], check_approved: bool = True, ) -> DbOrganisation: """Check if organisation name exists, else error. The org_id can also be an org name. """ + if not org_id: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail="Organisation id not provided", + ) + try: org_id = int(org_id) except ValueError: @@ -98,14 +129,14 @@ async def check_org_exists( log.debug(f"Getting organisation by id: {org_id}") db_organisation = await get_organisation_by_id(db, org_id, check_approved) - if isinstance(org_id, str): + else: # is string log.debug(f"Getting organisation by name: {org_id}") db_organisation = await get_organisation_by_name(db, org_id, check_approved) if not db_organisation: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, - detail=f"Organisation {org_id} does not exist", + detail=f"Organisation ({org_id}) does not exist", ) log.debug(f"Organisation match: {db_organisation}") diff --git a/src/backend/app/organisations/organisation_routes.py b/src/backend/app/organisations/organisation_routes.py index 6b5000f004..810a77e511 100644 --- a/src/backend/app/organisations/organisation_routes.py +++ b/src/backend/app/organisations/organisation_routes.py @@ -53,7 +53,6 @@ async def get_organisations( @router.get("/{org_id}", response_model=organisation_schemas.OrganisationOut) async def get_organisation_detail( organisation: DbOrganisation = Depends(org_exists), - db: Session = Depends(database.get_db), ): """Get a specific organisation by id or name.""" return organisation @@ -61,6 +60,7 @@ async def get_organisation_detail( @router.post("/", response_model=organisation_schemas.OrganisationOut) async def create_organisation( + # Depends required below to allow logo upload org: organisation_schemas.OrganisationIn = Depends(), logo: UploadFile = File(None), db: Session = Depends(database.get_db), diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index adf9dc6bca..5e10fa5002 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -21,44 +21,26 @@ from typing import Optional from fastapi import Form -from pydantic import BaseModel, Field, HttpUrl, SecretStr, computed_field -from pydantic.functional_validators import field_validator +from pydantic import BaseModel, Field, computed_field -from app.config import decrypt_value, encrypt_value +from app.config import HttpUrlStr from app.models.enums import OrganisationType +from app.projects.project_schemas import ODKCentralIn # class OrganisationBase(BaseModel): # """Base model for organisation to extend.""" -class OrganisationIn(BaseModel): +class OrganisationIn(ODKCentralIn): """Organisation to create from user input.""" name: str = Field(Form(..., description="Organisation name")) description: Optional[str] = Field( Form(None, description="Organisation description") ) - url: Optional[HttpUrl] = Field(Form(None, description=("Organisation website URL"))) - odk_central_url: Optional[str] = Field( - Form(None, description="Organisation default ODK URL") + url: Optional[HttpUrlStr] = Field( + Form(None, description="Organisation website URL") ) - odk_central_user: Optional[str] = Field( - Form(None, description="Organisation default ODK User") - ) - odk_central_password: Optional[SecretStr] = Field( - Form(None, description="Organisation default ODK Password") - ) - - @field_validator("url", mode="after") - @classmethod - def convert_url_to_str(cls, value: HttpUrl) -> str: - """Convert Pydantic Url type to string. - - Database models do not accept type Url for a string field. - """ - if value: - return value.unicode_string() - return "" @computed_field @property @@ -72,14 +54,6 @@ def slug(self) -> str: slug = sub(r"[-\s]+", "-", slug) return slug - @field_validator("odk_central_password", mode="before") - @classmethod - def encrypt_odk_password(cls, value: str) -> Optional[SecretStr]: - """Encrypt the ODK Central password before db insertion.""" - if not value: - return None - return SecretStr(encrypt_value(value)) - class OrganisationEdit(OrganisationIn): """Organisation to edit via user input.""" @@ -98,22 +72,3 @@ class OrganisationOut(BaseModel): slug: Optional[str] url: Optional[str] type: OrganisationType - odk_central_url: Optional[str] = None - - -class OrganisationOutWithCreds(BaseModel): - """Organisation plus ODK Central credentials. - - Note: the password is obsfucated as SecretStr. - """ - - odk_central_user: Optional[str] = None - odk_central_password: Optional[SecretStr] = None - - def model_post_init(self, ctx): - """Run logic after model object instantiated.""" - # Decrypt odk central password from database - if self.odk_central_password: - self.odk_central_password = SecretStr( - decrypt_value(self.odk_central_password.get_secret_value()) - ) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 65685963d3..9c225e02f9 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -57,6 +57,7 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from app.auth.osm import AuthUser from app.central import central_crud from app.config import encrypt_value, settings from app.db import db_models @@ -203,7 +204,9 @@ async def partial_update_project_info( async def update_project_info( - db: Session, project_metadata: project_schemas.ProjectUpload, project_id + db: Session, + project_metadata: project_schemas.ProjectUpload, + project_id: int, ): """Full project update for PUT.""" user = project_metadata.author @@ -251,35 +254,18 @@ async def update_project_info( async def create_project_with_project_info( - db: Session, project_metadata: project_schemas.ProjectUpload, odk_project_id: int + db: Session, + project_metadata: project_schemas.ProjectUpload, + odk_project_id: int, + current_user: AuthUser, ): """Create a new project, including all associated info.""" # FIXME the ProjectUpload model should be converted to the db model directly # FIXME we don't need to extract each variable and pass manually # project_data = project_metadata.model_dump() - project_user = project_metadata.author - project_info = project_metadata.project_info - xform_title = project_metadata.xform_title - odk_credentials = project_metadata.odk_central - hashtags = project_metadata.hashtags - organisation_id = project_metadata.organisation_id - task_split_type = project_metadata.task_split_type - task_split_dimension = project_metadata.task_split_dimension - task_num_buildings = project_metadata.task_num_buildings - data_extract_type = project_metadata.task_num_buildings + log.warning(project_metadata.model_dump()) - # verify data coming in - if not project_user: - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, - detail="User details are missing", - ) - if not project_info: - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, - detail="Project info is missing", - ) if not odk_project_id: raise HTTPException( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -288,71 +274,33 @@ async def create_project_with_project_info( log.debug( "Creating project in FMTM database with vars: " - f"project_user: {project_user} | " - f"project_info: {project_info} | " - f"xform_title: {xform_title} | " - f"hashtags: {hashtags}| " - f"organisation_id: {organisation_id}" + f"project_user: {current_user.username} | " + f"project_name: {project_metadata.project_info.name} | " + f"xform_title: {project_metadata.xform_title} | " + f"hashtags: {project_metadata.hashtags} | " + f"organisation_id: {project_metadata.organisation_id}" ) - # Check / set credentials - if odk_credentials: - url = odk_credentials.odk_central_url - user = odk_credentials.odk_central_user - pw = odk_credentials.odk_central_password.get_secret_value() - - else: - log.debug("ODKCentral connection variables not set in function") - log.debug("Attempting extraction from environment variables") - url = settings.ODK_CENTRAL_URL - user = settings.ODK_CENTRAL_USER - pw = settings.ODK_CENTRAL_PASSWD + # Extract project_info details, then remove key + project_name = project_metadata.project_info.name + project_description = project_metadata.project_info.description + project_short_description = project_metadata.project_info.short_description - # get db user - # TODO: get this from logged in user / request instead, - # return 403 (forbidden) if not authorized - db_user = await user_crud.get_user(db, project_user.id) - if not db_user: - raise HTTPException( - status_code=400, detail=f"User {project_user.username} does not exist" - ) - - hashtags = ( - list( - map( - lambda hashtag: hashtag if hashtag.startswith("#") else f"#{hashtag}", - hashtags, - ) - ) - if hashtags - else None - ) # create new project db_project = db_models.DbProject( - author=db_user, + author_id=current_user.id, odkid=odk_project_id, - project_name_prefix=project_info.name, - xform_title=xform_title, - odk_central_url=url, - odk_central_user=user, - odk_central_password=pw, - hashtags=hashtags, - organisation_id=organisation_id, - task_split_type=task_split_type, - task_split_dimension=task_split_dimension, - task_num_buildings=task_num_buildings, - data_extract_type=data_extract_type, - # country=[project_metadata.country], - # location_str=f"{project_metadata.city}, {project_metadata.country}", + project_name_prefix=project_name, + **project_metadata.model_dump(exclude=["project_info"]), ) db.add(db_project) # add project info (project id needed to create project info) db_project_info = db_models.DbProjectInfo( project=db_project, - name=project_info.name, - short_description=project_info.short_description, - description=project_info.description, + name=project_name, + short_description=project_short_description, + description=project_description, ) db.add(db_project_info) @@ -1180,7 +1128,7 @@ def generate_task_files( task_id: int, xlsform: str, form_type: str, - odk_credentials: project_schemas.ODKCentral, + odk_credentials: project_schemas.ODKCentralDecrypted, ): """Generate all files for a task.""" project_log = log.bind(task="create_project", project_id=project_id) @@ -1203,7 +1151,7 @@ def generate_task_files( appuser = OdkAppUser( odk_credentials.odk_central_url, odk_credentials.odk_central_user, - odk_credentials.odk_central_password.get_secret_value(), + odk_credentials.odk_central_password, ) appuser_json = appuser.create(odk_id, appuser_name) @@ -1379,7 +1327,7 @@ def generate_appuser_files( "odk_central_password": one.odk_central_password, } - odk_credentials = project_schemas.ODKCentral(**odk_credentials) + odk_credentials = project_schemas.ODKCentralDecrypted(**odk_credentials) xform_title = one.xform_title if one.xform_title else None @@ -1668,7 +1616,7 @@ async def convert_to_app_project(db_project: db_models.DbProject): return None log.debug("Converting db project to app project") - app_project: project_schemas.Project = db_project + app_project = db_project if db_project.outline: log.debug("Converting project outline to geojson") @@ -1892,7 +1840,7 @@ async def update_project_form( odk_id = project.odkid # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 0c0c3b4676..1c662f040c 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -89,7 +89,7 @@ async def get_projet_details(project_id: int, db: Session = Depends(database.get raise HTTPException(status_code=404, details={"Project not found"}) # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, @@ -215,15 +215,15 @@ async def read_project(project_id: int, db: Session = Depends(database.get_db)): @router.delete("/{project_id}") async def delete_project( project: db_models.DbProject = Depends(project_deps.get_project_by_id), - db: Session = Depends(database.get_db), current_user: AuthUser = Depends(login_required), + db: Session = Depends(database.get_db), ): """Delete a project from both ODK Central and the local database.""" log.info( f"User {current_user.username} attempting deletion of project {project.id}" ) # Odk crendentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, @@ -240,6 +240,7 @@ async def delete_project( @router.post("/create_project", response_model=project_schemas.ProjectOut) async def create_project( project_info: project_schemas.ProjectUpload, + current_user: AuthUser = Depends(login_required), db: Session = Depends(database.get_db), ): """Create a project in ODK Central and the local database. @@ -247,14 +248,39 @@ async def create_project( TODO refactor to standard REST POST to /projects TODO but first check doesn't break other endpoints """ - log.debug(f"Creating project {project_info.project_info.name}") + # Check if organisation exists + org = await organisation_deps.check_org_exists(db, project_info.organisation_id) + + log.info( + f"User {current_user.username} attempting creation of project " + f"{project_info.project_info.name}" + ) + + # Must decrypt ODK password & connect to ODK Central before proj created + if project_info.odk_central_url: + odk_creds_decrypted = project_schemas.ODKCentralDecrypted( + odk_central_url=project_info.odk_central_url, + odk_central_user=project_info.odk_central_user, + odk_central_password=project_info.odk_central_password, + ) + else: + # Use default org credentials if none passed + log.debug( + "No odk credentials passed during project creation. " + "Defaulting to organisation credentials." + ) + odk_creds_decrypted = await organisation_deps.get_org_odk_creds(org) odkproject = central_crud.create_odk_project( - project_info.project_info.name, project_info.odk_central + project_info.project_info.name, + odk_creds_decrypted, ) project = await project_crud.create_project_with_project_info( - db, project_info, odkproject["id"] + db, + project_info, + odkproject["id"], + current_user, ) if not project: raise HTTPException(status_code=404, detail="Project creation failed") @@ -266,6 +292,7 @@ async def create_project( async def update_project( id: int, project_info: project_schemas.ProjectUpload, + current_user: AuthUser = Depends(login_required), db: Session = Depends(database.get_db), ): """Update an existing project by ID. @@ -275,7 +302,6 @@ async def update_project( Parameters: - id: ID of the project to update - - author: Author username and id - project_info: Updated project information Returns: @@ -544,7 +570,7 @@ async def generate_files( xls_form_config_file: Optional[UploadFile] = File(None), data_extracts: Optional[UploadFile] = File(None), db: Session = Depends(database.get_db), - # current_user: AuthUser = Depends(login_required), + current_user: AuthUser = Depends(login_required), ): """Generate additional content to initialise the project. @@ -568,6 +594,7 @@ async def generate_files( xls_form_config_file (UploadFile, optional): The config YAML for the XLS form. data_extracts (UploadFile, optional): Custom data extract GeoJSON. db (Session): Database session, provided automatically. + current_user (AuthUser): Current logged in user. Returns: json (JSONResponse): A success message containing the project ID. diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index f7a02854e0..f23020aa44 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -19,45 +19,103 @@ import uuid from datetime import datetime -from typing import List, Optional, Union +from typing import Any, List, Optional, Union from dateutil import parser +from fastapi import Form from geojson_pydantic import Feature as GeojsonFeature -from pydantic import BaseModel, SecretStr +from loguru import logger as log +from pydantic import BaseModel, Field, computed_field from pydantic.functional_serializers import field_serializer -from pydantic.functional_validators import field_validator +from pydantic.functional_validators import field_validator, model_validator +from typing_extensions import Self -from app.config import decrypt_value, encrypt_value +from app.config import HttpUrlStr, decrypt_value, encrypt_value from app.db import db_models +from app.db.postgis_utils import geometry_to_geojson from app.models.enums import ProjectPriority, ProjectStatus, TaskSplitType from app.tasks import tasks_schemas from app.users.user_schemas import User -class ODKCentral(BaseModel): - """ODK Central credentials.""" +class ODKCentralIn(BaseModel): + """ODK Central credentials inserted to database.""" - odk_central_url: str - odk_central_user: str - odk_central_password: SecretStr + odk_central_url: Optional[HttpUrlStr] = Field( + Form(None, description="ODK Central URL") + ) + odk_central_user: Optional[str] = Field(Form(None, description="ODK Central User")) + odk_central_password: Optional[str] = Field( + Form(None, description="ODK Central Password") + ) - def model_post_init(self, ctx): - """Run logic after model object instantiated.""" - # Decrypt odk central password from database - self.odk_central_password = SecretStr( - decrypt_value(self.odk_central_password.get_secret_value()) - ) + @field_validator("odk_central_url", mode="after") + @classmethod + def remove_trailing_slash(cls, value: HttpUrlStr) -> Optional[HttpUrlStr]: + """Remove trailing slash from ODK Central URL.""" + if not value: + return None + if value.endswith("/"): + return value[:-1] + return value - @field_validator("odk_central_password", mode="before") + @model_validator(mode="after") + def all_odk_vars_together(self) -> Self: + """Ensure if one ODK variable is set, then all are.""" + if any( + [ + self.odk_central_url, + self.odk_central_user, + self.odk_central_password, + ] + ) and not all( + [ + self.odk_central_url, + self.odk_central_user, + self.odk_central_password, + ] + ): + err = "All ODK details are required together: url, user, password" + log.debug(err) + raise ValueError(err) + return self + + @field_validator("odk_central_password", mode="after") @classmethod - def encrypt_odk_password(cls, value: str) -> SecretStr: + def encrypt_odk_password(cls, value: str) -> Optional[str]: """Encrypt the ODK Central password before db insertion.""" - return SecretStr(encrypt_value(value)) + if not value: + return None + return encrypt_value(value) + - @field_validator("odk_central_url", mode="before") +class ODKCentralDecrypted(BaseModel): + """ODK Central credentials extracted from database. + + WARNING never return this as a response model. + WARNING or log to the terminal. + """ + + odk_central_url: Optional[HttpUrlStr] = None + odk_central_user: Optional[str] = None + odk_central_password: Optional[str] = None + + def model_post_init(self, ctx): + """Run logic after model object instantiated.""" + # Decrypt odk central password from database + if self.odk_central_password: + if isinstance(self.odk_central_password, str): + password = self.odk_central_password + else: + password = self.odk_central_password + self.odk_central_password = decrypt_value(password) + + @field_validator("odk_central_url", mode="after") @classmethod - def remove_trailing_slash(cls, value: str) -> str: + def remove_trailing_slash(cls, value: HttpUrlStr) -> HttpUrlStr: """Remove trailing slash from ODK Central URL.""" + if not value: + return "" if value.endswith("/"): return value[:-1] return value @@ -79,23 +137,43 @@ class ProjectUpdate(BaseModel): description: Optional[str] = None -class ProjectUpload(BaseModel): +class ProjectIn(BaseModel): """Upload new project.""" - author: User project_info: ProjectInfo - xform_title: Optional[str] - odk_central: ODKCentral + xform_title: str hashtags: Optional[List[str]] = None organisation_id: Optional[int] = None task_split_type: Optional[TaskSplitType] = None task_split_dimension: Optional[int] = None task_num_buildings: Optional[int] = None data_extract_type: Optional[str] = None - # city: str # country: str + @field_validator("hashtags", mode="after") + @classmethod + def prepend_hash_to_tags(cls, hashtags: List[str]) -> Optional[List[str]]: + """Add '#' to hashtag if missing. Also added default '#FMTM'.""" + if not hashtags: + return None + + hashtags_with_hash = [ + f"#{hashtag}" if hashtag and not hashtag.startswith("#") else hashtag + for hashtag in hashtags + ] + + if "#FMTM" not in hashtags_with_hash: + hashtags_with_hash.append("#FMTM") + + return hashtags_with_hash + + +class ProjectUpload(ProjectIn, ODKCentralIn): + """Project upload details, plus ODK credentials.""" + + pass + class Feature(BaseModel): """Features used for Task definitions.""" @@ -170,18 +248,27 @@ class PaginatedProjectSummaries(BaseModel): class ProjectBase(BaseModel): """Base project model.""" + outline: Any = Field(exclude=True) + id: int odkid: int author: User project_info: ProjectInfo status: ProjectStatus # location_str: str - outline_geojson: Optional[GeojsonFeature] = None project_tasks: Optional[List[tasks_schemas.Task]] xform_title: Optional[str] = None hashtags: Optional[List[str]] = None organisation_id: Optional[int] = None + @computed_field + @property + def outline_geojson(self) -> Optional[GeojsonFeature]: + """Sanitise the organisation name for use in a URL.""" + if not self.outline: + return None + return geometry_to_geojson(self.outline, {"id": self.id}, self.id) + class ProjectOut(ProjectBase): """Project display to user.""" diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index 327ca7729d..03f8b5b0df 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -72,7 +72,7 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None) ) # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -151,7 +151,7 @@ def convert_to_osm(db: Session, project_id: int, task_id: int): form_category = project_info.xform_title # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -235,7 +235,7 @@ def gather_all_submission_csvs(db, project_id): project_tasks = project_info.tasks # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -330,7 +330,7 @@ def update_submission_in_s3( project = get_project_sync(db, project_id) # Gather metadata - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, @@ -427,7 +427,7 @@ def get_all_submissions_json(db: Session, project_id): project_info = get_project_sync(db, project_id) # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -457,7 +457,7 @@ def get_all_submissions_json(db: Session, project_id): # project_tasks = project_info.tasks # # ODK Credentials -# odk_credentials = project_schemas.ODKCentral( +# odk_credentials = project_schemas.ODKCentralDecrypted( # odk_central_url=project_info.odk_central_url, # odk_central_user=project_info.odk_central_user, # odk_central_password=project_info.odk_central_password, @@ -499,7 +499,7 @@ async def download_submission( project_tasks = project_info.tasks # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -609,7 +609,7 @@ async def get_submission_points(db: Session, project_id: int, task_id: int = Non form_category = project_info.xform_title # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -674,7 +674,7 @@ async def get_submission_count_of_a_project(db: Session, project_id: int): project_tasks = project_info.tasks # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project_info.odk_central_url, odk_central_user=project_info.odk_central_user, odk_central_password=project_info.odk_central_password, @@ -818,7 +818,7 @@ async def get_submission_by_task( Returns: Tuple: A tuple containing the list of submissions and the count. """ - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index 4a0df6a699..90e49d4786 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -162,7 +162,7 @@ async def task_features_count( project = await project_crud.get_project(db, project_id) # ODK Credentials - odk_credentials = project_schemas.ODKCentral( + odk_credentials = project_schemas.ODKCentralDecrypted( odk_central_url=project.odk_central_url, odk_central_user=project.odk_central_user, odk_central_password=project.odk_central_password, diff --git a/src/backend/app/tasks/tasks_schemas.py b/src/backend/app/tasks/tasks_schemas.py index 3389c01bb0..ace91f1857 100644 --- a/src/backend/app/tasks/tasks_schemas.py +++ b/src/backend/app/tasks/tasks_schemas.py @@ -20,7 +20,7 @@ from datetime import datetime from typing import Any, List, Optional -from geojson_pydantic import Feature +from geojson_pydantic import Feature as GeojsonFeature from loguru import logger as log from pydantic import BaseModel, ConfigDict, Field, ValidationInfo from pydantic.functional_serializers import field_serializer @@ -71,8 +71,8 @@ class Task(BaseModel): project_id: int project_task_index: int project_task_name: str - outline_geojson: Optional[Feature] = None - outline_centroid: Optional[Feature] = None + outline_geojson: Optional[GeojsonFeature] = None + outline_centroid: Optional[GeojsonFeature] = None initial_feature_count: Optional[int] = None task_status: TaskStatus locked_by_uid: Optional[int] = None diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index 68bdcf7ae4..c67fc8a70f 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -29,14 +29,14 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists +from app.auth.osm import AuthUser from app.central import central_crud from app.config import settings from app.db.database import Base, get_db from app.db.db_models import DbOrganisation, DbUser from app.main import get_application from app.projects import project_crud -from app.projects.project_schemas import ODKCentral, ProjectInfo, ProjectUpload -from app.users.user_schemas import User +from app.projects.project_schemas import ODKCentralDecrypted, ProjectInfo, ProjectUpload engine = create_engine(settings.FMTM_DB_URL.unicode_string()) TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -102,6 +102,7 @@ def organisation(db): description="test org", url="https://test.org", logo="none", + approved=True, ) db.add(db_org) db.commit() @@ -112,32 +113,30 @@ def organisation(db): async def project(db, user, organisation): """A test project, using the test user and org.""" project_metadata = ProjectUpload( - author=User(username=user.username, id=user.id), project_info=ProjectInfo( name="test project", short_description="test", description="test", ), xform_title="buildings", - odk_central=ODKCentral( - odk_central_url=os.getenv("ODK_CENTRAL_URL"), - odk_central_user=os.getenv("ODK_CENTRAL_USER"), - odk_central_password=os.getenv("ODK_CENTRAL_PASSWD"), - ), + odk_central_url=os.getenv("ODK_CENTRAL_URL"), + odk_central_user=os.getenv("ODK_CENTRAL_USER"), + odk_central_password=os.getenv("ODK_CENTRAL_PASSWD"), hashtags=["hot-fmtm"], organisation_id=organisation.id, ) - # Create ODK Central Project - if project_metadata.odk_central.odk_central_url.endswith("/"): - # Remove trailing slash - project_metadata.odk_central.odk_central_url = ( - project_metadata.odk_central.odk_central_url[:-1] - ) + odk_creds_decrypted = ODKCentralDecrypted( + odk_central_url=project_metadata.odk_central_url, + odk_central_user=project_metadata.odk_central_user, + odk_central_password=project_metadata.odk_central_password, + ) + # Create ODK Central Project try: odkproject = central_crud.create_odk_project( - project_metadata.project_info.name, project_metadata.odk_central + project_metadata.project_info.name, + odk_creds_decrypted, ) log.debug(f"ODK project returned: {odkproject}") assert odkproject is not None @@ -148,7 +147,10 @@ async def project(db, user, organisation): # Create FMTM Project try: new_project = await project_crud.create_project_with_project_info( - db, project_metadata, odkproject["id"] + db, + project_metadata, + odkproject["id"], + AuthUser(username=user.username, id=user.id), ) log.debug(f"Project returned: {new_project.__dict__}") assert new_project is not None diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index 57f067da4a..a8a3e90fea 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -33,7 +33,7 @@ from shapely.geometry import shape from app.central.central_crud import create_odk_project -from app.config import settings +from app.config import encrypt_value, settings from app.db import db_models from app.projects import project_crud, project_schemas from app.tasks import tasks_crud @@ -41,27 +41,29 @@ odk_central_url = os.getenv("ODK_CENTRAL_URL") odk_central_user = os.getenv("ODK_CENTRAL_USER") -odk_central_password = os.getenv("ODK_CENTRAL_PASSWD") +odk_central_password = encrypt_value(os.getenv("ODK_CENTRAL_PASSWD", "")) -async def test_create_project(client, organisation, user): +async def test_create_project(client, organisation): """Test project creation endpoint.""" + odk_credentials = { + "odk_central_url": odk_central_url, + "odk_central_user": odk_central_user, + "odk_central_password": odk_central_password, + } + odk_credentials = project_schemas.ODKCentralDecrypted(**odk_credentials) + project_data = { - "author": {"username": user.username, "id": user.id}, "project_info": { "name": "test project", "short_description": "test", "description": "test", }, "xform_title": "buildings", - "odk_central": { - "odk_central_url": odk_central_url, - "odk_central_user": odk_central_user, - "odk_central_password": odk_central_password, - }, - "hashtags": ["hot-fmtm"], + "hashtags": ["#FMTM"], "organisation_id": organisation.id, } + project_data.update(**odk_credentials.model_dump()) response = client.post("/projects/create_project", json=project_data) @@ -133,7 +135,7 @@ async def test_generate_appuser_files(db, project): "odk_central_user": odk_central_user, "odk_central_password": odk_central_password, } - odk_credentials = project_schemas.ODKCentral(**odk_credentials) + odk_credentials = project_schemas.ODKCentralDecrypted(**odk_credentials) project_id = project.id log.debug(f"Testing project ID: {project_id}") diff --git a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js index 1e5ca13fac..8ee72bf2fe 100644 --- a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js +++ b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js @@ -164,7 +164,7 @@ const LayerSwitcherControl = ({ map, visible = 'osm' }) => { if ( location.pathname.includes('project_details') || location.pathname.includes('upload-area') || - location.pathname.includes('select-form') || + location.pathname.includes('select-category') || location.pathname.includes('data-extract') || location.pathname.includes('split-tasks') ) { diff --git a/src/frontend/src/components/createnewproject/DataExtract.tsx b/src/frontend/src/components/createnewproject/DataExtract.tsx index 99d0d96f8e..561c11eb07 100644 --- a/src/frontend/src/components/createnewproject/DataExtract.tsx +++ b/src/frontend/src/components/createnewproject/DataExtract.tsx @@ -141,7 +141,7 @@ const DataExtract = ({ flag, customLineUpload, setCustomLineUpload, customPolygo }, [formValues?.dataExtractWays, formValues?.dataExtractFeatureType]); const toggleStep = (step, url) => { - if (url === '/select-form') { + if (url === '/select-category') { dispatch( CreateProjectActions.SetIndividualProjectDetailsData({ ...formValues, @@ -326,7 +326,7 @@ const DataExtract = ({ flag, customLineUpload, setCustomLineUpload, customPolygo btnText="PREVIOUS" btnType="secondary" type="button" - onClick={() => toggleStep(3, '/select-form')} + onClick={() => toggleStep(3, '/select-category')} className="fmtm-font-bold" />