fix: map default odk credentials to organisations (#1123)

* fix: add odk cred org schemas for db and pydantic

* fix: add HttpUrlStr type for Url validation of str values

* fix: fix OdkCentral schema to handle passwords, inherit

* refactor: use new ODKCentralDecrypted, simplify proj create

* fix: add org_deps get_org_odk_creds method

* refactor(frontend): update create_project json structure

* refactor(frontend): remove code to add #FMTM tag automatically

* test: fix project route tests

* refactor: revert setting outline_geojson in create_app_project

* feat(frontend): option to add default odk creds for organisation

* refactor(frontend): rename form during project creation --> category

* refactor: remove additional route added during rebase

src/backend/app/central/central_crud.py

@@ -35,12 +35,12 @@
 from app.projects import project_schemas
 
 
-def get_odk_project(odk_central: project_schemas.ODKCentral = None):
+def get_odk_project(odk_central: project_schemas.ODKCentralDecrypted = None):
     """Helper function to get the OdkProject with credentials."""
     if odk_central:
         url = odk_central.odk_central_url
         user = odk_central.odk_central_user
-        pw = odk_central.odk_central_password.get_secret_value()
+        pw = odk_central.odk_central_password
     else:
         log.debug("ODKCentral connection variables not set in function")
         log.debug("Attempting extraction from environment variables")
@@ -60,12 +60,12 @@ def get_odk_project(odk_central: project_schemas.ODKCentral = None):
     return project
 
 
-def get_odk_form(odk_central: project_schemas.ODKCentral = None):
+def get_odk_form(odk_central: project_schemas.ODKCentralDecrypted = None):
     """Helper function to get the OdkForm with credentials."""
     if odk_central:
         url = odk_central.odk_central_url
         user = odk_central.odk_central_user
-        pw = odk_central.odk_central_password.get_secret_value()
+        pw = odk_central.odk_central_password
 
     else:
         log.debug("ODKCentral connection variables not set in function")
@@ -86,12 +86,12 @@ def get_odk_form(odk_central: project_schemas.ODKCentral = None):
     return form
 
 
-def get_odk_app_user(odk_central: project_schemas.ODKCentral = None):
+def get_odk_app_user(odk_central: project_schemas.ODKCentralDecrypted = None):
     """Helper function to get the OdkAppUser with credentials."""
     if odk_central:
         url = odk_central.odk_central_url
         user = odk_central.odk_central_user
-        pw = odk_central.odk_central_password.get_secret_value()
+        pw = odk_central.odk_central_password
     else:
         log.debug("ODKCentral connection variables not set in function")
         log.debug("Attempting extraction from environment variables")
@@ -111,13 +111,15 @@ def get_odk_app_user(odk_central: project_schemas.ODKCentral = None):
     return form
 
 
-def list_odk_projects(odk_central: project_schemas.ODKCentral = None):
+def list_odk_projects(odk_central: project_schemas.ODKCentralDecrypted = None):
     """List all projects on a remote ODK Server."""
     project = get_odk_project(odk_central)
     return project.listProjects()
 
 
-def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None):
+def create_odk_project(
+    name: str, odk_central: project_schemas.ODKCentralDecrypted = None
+):
     """Create a project on a remote ODK Server."""
     project = get_odk_project(odk_central)
 
@@ -144,7 +146,7 @@ def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None
 
 
 async def delete_odk_project(
-    project_id: int, odk_central: project_schemas.ODKCentral = None
+    project_id: int, odk_central: project_schemas.ODKCentralDecrypted = None
 ):
     """Delete a project from a remote ODK Server."""
     # FIXME: when a project is deleted from Central, we have to update the
@@ -159,7 +161,7 @@ async def delete_odk_project(
 
 
 def delete_odk_app_user(
-    project_id: int, name: str, odk_central: project_schemas.ODKCentral = None
+    project_id: int, name: str, odk_central: project_schemas.ODKCentralDecrypted = None
 ):
     """Delete an app-user from a remote ODK Server."""
     odk_app_user = get_odk_app_user(odk_central)
@@ -202,7 +204,7 @@ def create_odk_xform(
     project_id: int,
     xform_id: str,
     filespec: str,
-    odk_credentials: project_schemas.ODKCentral = None,
+    odk_credentials: project_schemas.ODKCentralDecrypted = None,
     create_draft: bool = False,
     upload_media=True,
     convert_to_draft_when_publishing=True,
@@ -213,7 +215,7 @@ def create_odk_xform(
     # Pass odk credentials of project in xform
 
     if not odk_credentials:
-        odk_credentials = project_schemas.ODKCentral(
+        odk_credentials = project_schemas.ODKCentralDecrypted(
             odk_central_url=settings.ODK_CENTRAL_URL,
             odk_central_user=settings.ODK_CENTRAL_USER,
             odk_central_password=settings.ODK_CENTRAL_PASSWD,
@@ -245,7 +247,7 @@ def delete_odk_xform(
     project_id: int,
     xform_id: str,
     filespec: str,
-    odk_central: project_schemas.ODKCentral = None,
+    odk_central: project_schemas.ODKCentralDecrypted = None,
 ):
     """Delete an XForm from a remote ODK Central server."""
     xform = get_odk_form(odk_central)
@@ -256,7 +258,7 @@ def delete_odk_xform(
 
 def list_odk_xforms(
     project_id: int,
-    odk_central: project_schemas.ODKCentral = None,
+    odk_central: project_schemas.ODKCentralDecrypted = None,
     metadata: bool = False,
 ):
     """List all XForms in an ODK Central project."""
@@ -267,7 +269,7 @@ def list_odk_xforms(
 
 
 def get_form_full_details(
-    odk_project_id: int, form_id: str, odk_central: project_schemas.ODKCentral
+    odk_project_id: int, form_id: str, odk_central: project_schemas.ODKCentralDecrypted
 ):
     """Get additional metadata for ODK Form."""
     form = get_odk_form(odk_central)
@@ -276,15 +278,17 @@ def get_form_full_details(
 
 
 def get_odk_project_full_details(
-    odk_project_id: int, odk_central: project_schemas.ODKCentral
+    odk_project_id: int, odk_central: project_schemas.ODKCentralDecrypted
 ):
     """Get additional metadata for ODK project."""
     project = get_odk_project(odk_central)
     project_details = project.getFullDetails(odk_project_id)
     return project_details
 
 
-def list_submissions(project_id: int, odk_central: project_schemas.ODKCentral = None):
+def list_submissions(
+    project_id: int, odk_central: project_schemas.ODKCentralDecrypted = None
+):
     """List all submissions for a project, aggregated from associated users."""
     project = get_odk_project(odk_central)
     xform = get_odk_form(odk_central)
@@ -326,7 +330,7 @@ def download_submissions(
     xform_id: str,
     submission_id: str = None,
     get_json: bool = True,
-    odk_central: project_schemas.ODKCentral = None,
+    odk_central: project_schemas.ODKCentralDecrypted = None,
 ):
     """Download all submissions for an XForm."""
     xform = get_odk_form(odk_central)
@@ -506,7 +510,7 @@ def upload_media(
     project_id: int,
     xform_id: str,
     filespec: str,
-    odk_central: project_schemas.ODKCentral = None,
+    odk_central: project_schemas.ODKCentralDecrypted = None,
 ):
     """Upload a data file to Central."""
     xform = get_odk_form(odk_central)
@@ -517,7 +521,7 @@ def download_media(
     project_id: int,
     xform_id: str,
     filespec: str,
-    odk_central: project_schemas.ODKCentral = None,
+    odk_central: project_schemas.ODKCentralDecrypted = None,
 ):
     """Upload a data file to Central."""
     xform = get_odk_form(odk_central)

src/backend/app/central/central_routes.py

@@ -211,7 +211,7 @@ async def get_submission(
             return {"error": "No such project!"}
 
         # ODK Credentials
-        odk_credentials = project_schemas.ODKCentral(
+        odk_credentials = project_schemas.ODKCentralDecrypted(
             odk_central_url=first.odk_central_url,
             odk_central_user=first.odk_central_user,
             odk_central_password=first.odk_central_password,

src/backend/app/config.py

@@ -19,12 +19,17 @@
 
 import base64
 from functools import lru_cache
-from typing import Any, Optional, Union
+from typing import Annotated, Any, Optional, Union
 
 from cryptography.fernet import Fernet
-from pydantic import PostgresDsn, ValidationInfo, field_validator
+from pydantic import BeforeValidator, TypeAdapter, ValidationInfo, field_validator
+from pydantic.networks import HttpUrl, PostgresDsn
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+HttpUrlStr = Annotated[
+    str, BeforeValidator(lambda value: str(TypeAdapter(HttpUrl).validate_python(value)))
+]
+
 
 class Settings(BaseSettings):
     """Main settings class, defining environment variables."""
@@ -100,14 +105,14 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any:
         )
         return pg_url
 
-    ODK_CENTRAL_URL: Optional[str] = ""
+    ODK_CENTRAL_URL: Optional[HttpUrlStr] = ""
     ODK_CENTRAL_USER: Optional[str] = ""
     ODK_CENTRAL_PASSWD: Optional[str] = ""
 
     OSM_CLIENT_ID: str
     OSM_CLIENT_SECRET: str
     OSM_SECRET_KEY: str
-    OSM_URL: str = "https://www.openstreetmap.org"
+    OSM_URL: HttpUrlStr = "https://www.openstreetmap.org"
     OSM_SCOPE: str = "read_prefs"
     OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:7051/osmauth/"
 
@@ -147,7 +152,7 @@ def configure_s3_download_root(cls, v: Optional[str], info: ValidationInfo) -> s
                 return f"http://s3.{fmtm_domain}:{dev_port}"
             return f"https://s3.{fmtm_domain}"
 
-    UNDERPASS_API_URL: str = "https://api-prod.raw-data.hotosm.org/v1/"
+    UNDERPASS_API_URL: HttpUrlStr = "https://api-prod.raw-data.hotosm.org/v1/"
     SENTRY_DSN: Optional[str] = None
 
     model_config = SettingsConfigDict(

src/backend/app/organisations/organisation_crud.py

@@ -98,7 +98,7 @@ async def create_organisation(
     if await get_organisation_by_name(db, org_name=org_model.name):
         raise HTTPException(
             status_code=HTTPStatus.CONFLICT,
-            detail=f"Organisation already exists with the name {org_model.name}",
+            detail=f"Organisation already exists with the name ({org_model.name})",
         )
 
     # Required to check if exists on error

src/backend/app/organisations/organisation_deps.py

@@ -23,13 +23,12 @@
 from fastapi import Depends
 from fastapi.exceptions import HTTPException
 from loguru import logger as log
-from sqlalchemy import func
 from sqlalchemy.orm import Session
 
 from app.db.database import get_db
 from app.db.db_models import DbOrganisation, DbProject
 from app.models.enums import HTTPStatus
-from app.projects import project_deps
+from app.projects import project_deps, project_schemas
 
 
 async def get_organisation_by_name(
@@ -45,16 +44,20 @@ async def get_organisation_by_name(
     Returns:
         DbOrganisation: organisation with the given id
     """
-    org_obj = (
-        db.query(DbOrganisation)
-        .filter(func.lower(DbOrganisation.name).like(func.lower(f"%{org_name}%")))
-        .first()
-    )
+    # # For getting org with LIKE match
+    # org_obj = (
+    #     db.query(DbOrganisation)
+    #     .filter(func.lower(DbOrganisation.name).like(func.lower(f"%{org_name}%")))
+    #     .first()
+    # )
+    org_obj = db.query(DbOrganisation).filter_by(name=org_name).first()
+
     if org_obj and check_approved and org_obj.approved is False:
         raise HTTPException(
-            status_code=HTTPStatus.NOT_FOUND,
+            status_code=HTTPStatus.FORBIDDEN,
             detail=f"Organisation ({org_obj.id}) is not approved yet",
         )
+
     return org_obj
 
 
@@ -72,23 +75,51 @@ async def get_organisation_by_id(
         DbOrganisation: organisation with the given id
     """
     org_obj = db.query(DbOrganisation).filter_by(id=org_id).first()
+
     if org_obj and check_approved and org_obj.approved is False:
         raise HTTPException(
-            status_code=HTTPStatus.NOT_FOUND,
-            detail=f"Organisation {org_id} is not approved yet",
+            status_code=HTTPStatus.FORBIDDEN,
+            detail=f"Organisation ({org_id}) is not approved yet",
         )
     return org_obj
 
 
+async def get_org_odk_creds(
+    org: DbOrganisation,
+) -> project_schemas.ODKCentralDecrypted:
+    """Get odk credentials for an organisation, else error."""
+    url = org.odk_central_url
+    user = org.odk_central_user
+    password = org.odk_central_password
+
+    if not all([url, user, password]):
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND,
+            detail="Organisation does not have ODK Central credentials configured",
+        )
+
+    return project_schemas.ODKCentralDecrypted(
+        odk_central_url=org.odk_central_url,
+        odk_central_user=org.odk_central_user,
+        odk_central_password=org.odk_central_password,
+    )
+
+
 async def check_org_exists(
     db: Session,
-    org_id: Union[str, int],
+    org_id: Union[str, int, None],
     check_approved: bool = True,
 ) -> DbOrganisation:
     """Check if organisation name exists, else error.
 
     The org_id can also be an org name.
     """
+    if not org_id:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND,
+            detail="Organisation id not provided",
+        )
+
     try:
         org_id = int(org_id)
     except ValueError:
@@ -98,14 +129,14 @@ async def check_org_exists(
         log.debug(f"Getting organisation by id: {org_id}")
         db_organisation = await get_organisation_by_id(db, org_id, check_approved)
 
-    if isinstance(org_id, str):
+    else:  # is string
         log.debug(f"Getting organisation by name: {org_id}")
         db_organisation = await get_organisation_by_name(db, org_id, check_approved)
 
     if not db_organisation:
         raise HTTPException(
             status_code=HTTPStatus.NOT_FOUND,
-            detail=f"Organisation {org_id} does not exist",
+            detail=f"Organisation ({org_id}) does not exist",
         )
 
     log.debug(f"Organisation match: {db_organisation}")

src/backend/app/organisations/organisation_routes.py

@@ -53,14 +53,14 @@ async def get_organisations(
 @router.get("/{org_id}", response_model=organisation_schemas.OrganisationOut)
 async def get_organisation_detail(
     organisation: DbOrganisation = Depends(org_exists),
-    db: Session = Depends(database.get_db),
 ):
     """Get a specific organisation by id or name."""
     return organisation
 
 
 @router.post("/", response_model=organisation_schemas.OrganisationOut)
 async def create_organisation(
+    # Depends required below to allow logo upload
     org: organisation_schemas.OrganisationIn = Depends(),
     logo: UploadFile = File(None),
     db: Session = Depends(database.get_db),