From 5d0546f1d107902a1f858809df3b0b30c2d51207 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sat, 13 Jan 2024 02:36:30 +0000 Subject: [PATCH] refactor: fix all linting errors for code --- src/backend/app/auth/auth_routes.py | 1 + src/backend/app/central/central_crud.py | 6 +- src/backend/app/central/central_routes.py | 48 +-- src/backend/app/central/central_schemas.py | 12 + src/backend/app/db/db_models.py | 7 +- src/backend/app/models/enums.py | 1 + .../app/models/languages_and_countries.py | 10 +- src/backend/app/pagination/pagination.py | 4 + src/backend/app/projects/project_crud.py | 30 +- src/backend/app/projects/project_routes.py | 132 ++++---- src/backend/app/projects/project_schemas.py | 39 ++- src/backend/app/s3.py | 2 +- src/backend/app/submission/submission_crud.py | 290 +++++------------- .../app/submission/submission_routes.py | 123 ++++---- .../app/submission/submission_schemas.py | 2 + src/backend/app/tasks/tasks_crud.py | 36 ++- src/backend/app/tasks/tasks_routes.py | 24 +- src/backend/app/users/user_crud.py | 4 +- src/backend/app/users/user_routes.py | 6 +- src/backend/tests/__init__.py | 1 + 20 files changed, 374 insertions(+), 404 deletions(-) diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index 205294f224..6dedf7936f 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -133,6 +133,7 @@ async def my_data( """Read access token and get user details from OSM. Args: + request: The HTTP request (automatically included variable). db: The db session. user_data: User data provided by osm-login-python Auth. diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index e0a2ee6ba1..4a1f104aa4 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -33,9 +33,9 @@ from pyxform.xls2xform import xls2xform_convert from sqlalchemy.orm import Session -from ..config import settings -from ..db import db_models -from ..projects import project_schemas +from app.config import settings +from app.db import db_models +from app.projects import project_schemas def get_odk_project(odk_central: project_schemas.ODKCentral = None): diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 60933cf45f..2bc58206c6 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Routes to relay requests to ODK Central server.""" + import json from fastapi import APIRouter, Depends, HTTPException @@ -29,9 +31,9 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import text -from ..central import central_crud -from ..db import database -from ..projects import project_crud, project_schemas +from app.central import central_crud +from app.db import database +from app.projects import project_crud, project_schemas router = APIRouter( prefix="/central", @@ -75,17 +77,21 @@ async def create_appuser( async def get_form_lists( db: Session = Depends(database.get_db), skip: int = 0, limit: int = 100 ): - """This function retrieves a list of XForms from a database, - with the option to skip a certain number of records and limit the number of records returned. + """Get a list of all XForms on ODK Central. + + Option to skip a certain number of records and limit the number of + records returned. Parameters: - skip:int: the number of records to skip before starting to retrieve records. Defaults to 0 if not provided. - limit:int: the maximum number of records to retrieve. Defaults to 10 if not provided. + skip (int): the number of records to skip before starting to retrieve records. + Defaults to 0 if not provided. + limit (int): the maximum number of records to retrieve. + Defaults to 10 if not provided. Returns: - A list of dictionary containing the id and title of each XForm record retrieved from the database. + list[dict]: list of id:title dicts of each XForm record. """ # NOTE runs in separate thread using run_in_threadpool forms = await run_in_threadpool(lambda: central_crud.get_form_list(db, skip, limit)) @@ -138,7 +144,8 @@ async def list_submissions( project_id: int, xml_form_id: str = None, db: Session = Depends(database.get_db), -): +) -> list[dict]: + """Get all submissions JSONs for a project.""" try: project = table( "projects", @@ -188,20 +195,21 @@ async def list_submissions( @router.get("/submission") async def get_submission( project_id: int, - xmlFormId: str = None, + xml_form_id: str = None, submission_id: str = None, db: Session = Depends(database.get_db), -): - """This api returns the submission json. +) -> dict: + """Return the submission JSON for a single XForm. Parameters: - project_id:int the id of the project in the database. - xml_form_id:str: the xmlFormId of the form in Central. - submission_id:str: the submission id of the submission in Central. + project_id (int): the id of the project in the database. + xml_form_id (str): the xml_form_id of the form in Central. + submission_id (str): the submission id of the submission in Central. If the submission_id is provided, an individual submission is returned. - Returns: Submission json. + Returns: + dict: Submission JSON. """ try: """Download the submissions data from Central.""" @@ -231,9 +239,9 @@ async def get_submission( submissions = [] - if xmlFormId and submission_id: + if xml_form_id and submission_id: data = central_crud.download_submissions( - first.odkid, xmlFormId, submission_id, True, odk_credentials + first.odkid, xml_form_id, submission_id, True, odk_credentials ) if submissions != 0: submissions.append(json.loads(data[0])) @@ -242,7 +250,7 @@ async def get_submission( submissions.append(json.loads(data[entry])) else: - if not xmlFormId: + if not xml_form_id: xforms = central_crud.list_odk_xforms(first.odkid, odk_credentials) for xform in xforms: try: @@ -262,7 +270,7 @@ async def get_submission( submissions.append(json.loads(data[entry])) else: data = central_crud.download_submissions( - first.odkid, xmlFormId, None, True, odk_credentials + first.odkid, xml_form_id, None, True, odk_credentials ) submissions.append(json.loads(data[0])) if len(data) >= 2: diff --git a/src/backend/app/central/central_schemas.py b/src/backend/app/central/central_schemas.py index d4157ac229..c9aa2a676c 100644 --- a/src/backend/app/central/central_schemas.py +++ b/src/backend/app/central/central_schemas.py @@ -15,28 +15,40 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Schemas for returned ODK Central objects.""" + from enum import Enum from pydantic import BaseModel class CentralBase(BaseModel): + """ODK Central return.""" + central_url: str class Central(CentralBase): + """ODK Central return, with extras.""" + geometry_geojson: str # qr_code_binary: bytes class CentralOut(CentralBase): + """ODK Central output.""" + pass class CentralFileType(BaseModel): + """ODK Central file return.""" + filetype: Enum("FileType", ["xform", "extract", "zip", "xlsform", "all"]) pass class CentralDetails(CentralBase): + """ODK Central details.""" + pass diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 2111da251d..cf14d49d2b 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -43,6 +43,8 @@ relationship, ) +from app.db.database import Base, FmtmMetadata +from app.db.postgis_utils import timestamp from app.models.enums import ( BackgroundTaskStatus, MappingLevel, @@ -60,9 +62,6 @@ ValidationPermission, ) -from .database import Base, FmtmMetadata -from .postgis_utils import timestamp - class DbUserRoles(Base): """Fine grained user access for projects, described by roles.""" @@ -448,7 +447,7 @@ class DbProject(Base): DbProjectInfo, cascade="all, delete, delete-orphan", uselist=False, - backref="projects", + backref="project", ) location_str = Column(String) diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index c5d9d8fad1..d71e9eb98c 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -34,6 +34,7 @@ class IntEnum(int, Enum): class HTTPStatus(IntEnum): """All HTTP status codes used in endpoints.""" + # Success OK = 200 CREATED = 201 diff --git a/src/backend/app/models/languages_and_countries.py b/src/backend/app/models/languages_and_countries.py index 27f97890f1..d109fe937f 100644 --- a/src/backend/app/models/languages_and_countries.py +++ b/src/backend/app/models/languages_and_countries.py @@ -1,4 +1,7 @@ -# see https://gist.github.com/alexanderjulo/4073388 +"""Language and country codes for reference. + +see https://gist.github.com/alexanderjulo/4073388 +""" languages = [ ("aa", "Afar"), @@ -34,7 +37,10 @@ ("zh", "Chinese"), ( "cu", - "Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic", + ( + "Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; " + "Old Church Slavonic", + ), ), ("cv", "Chuvash"), ("kw", "Cornish"), diff --git a/src/backend/app/pagination/pagination.py b/src/backend/app/pagination/pagination.py index 2c678e7939..3b5abc8c37 100644 --- a/src/backend/app/pagination/pagination.py +++ b/src/backend/app/pagination/pagination.py @@ -1,8 +1,11 @@ +"""Logic for API pagination.""" + import math from typing import List def get_pages_nav(total_pages, current_page): + """Get page position (prev / next pages).""" next_page = None prev_page = None if current_page + 1 <= total_pages: @@ -13,6 +16,7 @@ def get_pages_nav(total_pages, current_page): def paginate_data(data: List[dict], page_no: int, page_size: int, total_content: int): + """Generate pagination JSON.""" total_pages = math.ceil(total_content / page_size) next_page, prev_page = get_pages_nav(total_pages, page_no) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 2efdcb29e8..295fedf945 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -2064,22 +2064,6 @@ async def update_project_form( return True -async def update_odk_credentials_in_db( - project_instance: project_schemas.ProjectUpload, - odk_central_cred: project_schemas.ODKCentral, - odkid: int, - db: Session, -): - """Update odk credentials for a project.""" - project_instance.odkid = odkid - project_instance.odk_central_url = odk_central_cred.odk_central_url - project_instance.odk_central_user = odk_central_cred.odk_central_user - project_instance.odk_central_password = odk_central_cred.odk_central_password - - db.commit() - db.refresh(project_instance) - - async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str): """Get the geojson of those features for this project.""" query = text( @@ -2358,17 +2342,17 @@ async def get_tasks_count(db: Session, project_id: int): async def get_pagination(page: int, count: int, results_per_page: int, total: int): """Pagination result for splash page.""" total_pages = (count + results_per_page - 1) // results_per_page - hasNext = (page * results_per_page) < count # noqa: N806 - hasPrev = page > 1 # noqa: N806 + has_next = (page * results_per_page) < count # noqa: N806 + has_prev = page > 1 # noqa: N806 pagination = project_schemas.PaginationInfo( - hasNext=hasNext, - hasPrev=hasPrev, - nextNum=page + 1 if hasNext else None, + has_next=has_next, + has_prev=has_prev, + next_num=page + 1 if has_next else None, page=page, pages=total_pages, - prevNum=page - 1 if hasPrev else None, - perPage=results_per_page, + prev_num=page - 1 if has_prev else None, + per_page=results_per_page, total=total, ) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 62d99e2e65..10165570dd 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -40,16 +40,17 @@ from osm_fieldwork.make_data_extract import getChoices from osm_fieldwork.xlsforms import xlsforms_path from sqlalchemy.orm import Session +from sqlalchemy.sql import text from app.auth.osm import AuthUser, login_required +from app.central import central_crud +from app.db import database, db_models +from app.models.enums import TILES_FORMATS, TILES_SOURCE, HTTPStatus +from app.projects import project_crud, project_deps, project_schemas +from app.projects.project_crud import check_crs +from app.static import data_path from app.submission import submission_crud - -from ..central import central_crud -from ..db import database, db_models -from ..models.enums import TILES_FORMATS, TILES_SOURCE -from ..tasks import tasks_crud -from . import project_crud, project_schemas -from .project_crud import check_crs +from app.tasks import tasks_crud router = APIRouter( prefix="/projects", @@ -66,6 +67,7 @@ async def read_projects( limit: int = 100, db: Session = Depends(database.get_db), ): + """Return all projects.""" project_count, projects = await project_crud.get_projects(db, user_id, skip, limit) return projects @@ -117,6 +119,10 @@ async def get_projet_details(project_id: int, db: Session = Depends(database.get @router.post("/near_me", response_model=list[project_schemas.ProjectSummary]) async def get_tasks_near_me(lat: float, long: float, user_id: int = None): + """Get projects near me. + + TODO to be implemented in future. + """ return [project_schemas.ProjectSummary()] @@ -128,6 +134,7 @@ async def read_project_summaries( results_per_page: int = Query(13, le=100), db: Session = Depends(database.get_db), ): + """Get a paginated summary of projects.""" if hashtags: hashtags = hashtags.split(",") # create list of hashtags hashtags = list( @@ -167,6 +174,7 @@ async def search_project( results_per_page: int = Query(13, le=100), db: Session = Depends(database.get_db), ): + """Search projects by string, hashtag, or other criteria.""" if hashtags: hashtags = hashtags.split(",") # create list of hashtags hashtags = list( @@ -197,6 +205,7 @@ async def search_project( @router.get("/{project_id}", response_model=project_schemas.ReadProject) async def read_project(project_id: int, db: Session = Depends(database.get_db)): + """Get a specific project by ID.""" project = await project_crud.get_project_by_id(db, project_id) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -228,7 +237,11 @@ async def create_project( project_info: project_schemas.ProjectUpload, db: Session = Depends(database.get_db), ): - """Create a project in ODK Central and the local database.""" + """Create a project in ODK Central and the local database. + + TODO refactor to standard REST POST to /projects + TODO but first check doesn't break other endpoints + """ log.debug(f"Creating project {project_info.project_info.name}") if project_info.odk_central.odk_central_url.endswith("/"): @@ -251,28 +264,6 @@ async def create_project( return project -@router.post("/update_odk_credentials") -async def update_odk_credentials( - odk_central_cred: project_schemas.ODKCentral, - project_id: int, - db: Session = Depends(database.get_db), -): - """Update odk credential of a project.""" - if odk_central_cred.odk_central_url.endswith("/"): - odk_central_cred.odk_central_url = odk_central_cred.odk_central_url[:-1] - - project = await project_crud.get_project(db, project_id) - - if not project: - raise HTTPException(status_code=404, detail="Project not found") - - await project_crud.update_odk_credentials_in_db( - project, odk_central_cred, odkproject["id"], db - ) - - return JSONResponse(status_code=200, content={"success": True}) - - @router.put("/{id}", response_model=project_schemas.ProjectOut) async def update_project( id: int, @@ -336,9 +327,11 @@ async def upload_custom_xls( db: Session = Depends(database.get_db), ): """Upload a custom XLSForm to the database. - Parameters: - - upload: the XLSForm file - - category: the category of the XLSForm. + + Args: + upload (UploadFile): the XLSForm file + category (str): the category of the XLSForm. + db (Session): the DB session, provided automatically. """ content = await upload.read() # read file content name = upload.filename.split(".")[0] # get name of file without extension @@ -365,9 +358,7 @@ async def upload_custom_task_boundaries( Returns: dict: JSON containing success message, project ID, and number of tasks. """ - log.debug( - f"Uploading project boundary multipolygon for project ID: {project_id}" - ) + log.debug(f"Uploading project boundary multipolygon for project ID: {project_id}") # read entire file content = await project_geojson.read() boundary = json.loads(content) @@ -444,14 +435,14 @@ async def upload_project_boundary( ): """Uploads the project boundary. The boundary is uploaded as a geojson file. - Params: - - project_id (int): The ID of the project to update. - - boundary_geojson (UploadFile): The boundary file to upload. - - dimension (int): The new dimension of the project. - - db (Session): The database session to use. + Args: + project_id (int): The ID of the project to update. + boundary_geojson (UploadFile): The boundary file to upload. + dimension (int): The new dimension of the project. + db (Session): The database session to use. Returns: - - Dict: A dictionary with a message, the project ID, and the number of tasks in the project. + dict: JSON with message, project ID, and task count for project. """ # Validating for .geojson File. file_name = os.path.splitext(boundary_geojson.filename) @@ -493,6 +484,7 @@ async def edit_project_boundary( dimension: int = Form(500), db: Session = Depends(database.get_db), ): + """Edit the existing project boundary.""" # Validating for .geojson File. file_name = os.path.splitext(boundary_geojson.filename) file_ext = file_name[1] @@ -555,22 +547,28 @@ async def generate_files( data_extracts: Optional[UploadFile] = File(None), db: Session = Depends(database.get_db), ): - """Generate additional content for the project to function. + """Generate additional content to initialise the project. - QR codes, + Boundary, ODK Central forms, QR codes, etc. Accepts a project ID, category, custom form flag, and an uploaded file as inputs. The generated files are associated with the project ID and stored in the database. - This api generates qr_code, forms. This api also creates an app user for each task and provides the required roles. - Some of the other functionality of this api includes converting a xls file provided by the user to the xform, - generates osm data extracts and uploads it to the form. + This api generates qr_code, forms. This api also creates an app user for + each task and provides the required roles. + Some of the other functionality of this api includes converting a xls file + provided by the user to the xform, generates osm data extracts and uploads + it to the form. Args: + background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically. project_id (int): The ID of the project for which files are being generated. - polygon (bool): A boolean flag indicating whether the polygon + extract_polygon (bool): A boolean flag indicating whether the polygon is extracted or not. xls_form_upload (UploadFile, optional): A custom XLSForm to use in the project. A file should be provided if user wants to upload a custom xls form. + xls_form_config_file (UploadFile, optional): The config YAML for the XLS form. + data_extracts (UploadFile, optional): Custom data extract GeoJSON. + db (Session): Database session, provided automatically. Returns: json (JSONResponse): A success message containing the project ID. @@ -659,6 +657,7 @@ async def update_project_form( form: Optional[UploadFile], db: Session = Depends(database.get_db), ): + """Update XLSForm for a project.""" file_name = os.path.splitext(form.filename) file_ext = file_name[1] allowed_extensions = [".xls"] @@ -686,6 +685,7 @@ async def get_project_features( Args: project_id (int): The project id. task_id (int): The task id. + db (Session): the DB session, provided automatically. Returns: feature(json): JSON object containing a list of features @@ -701,9 +701,11 @@ async def generate_log( r"""Get the contents of a log file in a log format. ### Response - - **200 OK**: Returns the contents of the log file in a log format. Each line is separated by a newline character "\n". + - **200 OK**: Returns the contents of the log file in a log format. + Each line is separated by a newline character "\n". - - **500 Internal Server Error**: Returns an error message if the log file cannot be generated. + - **500 Internal Server Error**: Returns an error message if the log file + cannot be generated. ### Return format Task Status and Logs are returned in a JSON format. @@ -840,6 +842,7 @@ async def upload_custom_extract( @router.get("/download_form/{project_id}/") async def download_form(project_id: int, db: Session = Depends(database.get_db)): + """Download the XLSForm for a project.""" project = await project_crud.get_project(db, project_id) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -866,6 +869,10 @@ async def update_project_category( upload: Optional[UploadFile] = File(None), db: Session = Depends(database.get_db), ): + """Update the XLSForm category for a project. + + Not valid for custom form uploads. + """ contents = None project = await project_crud.get_project(db, project_id) @@ -905,6 +912,7 @@ async def update_project_category( @router.get("/download_template/") async def download_template(category: str, db: Session = Depends(database.get_db)): + """Download an XLSForm template to fill out.""" xlsform_path = f"{xlsforms_path}/{category}.xls" if os.path.exists(xlsform_path): return FileResponse(xlsform_path, filename="form.xls") @@ -921,6 +929,7 @@ async def download_project_boundary( Args: project_id (int): The id of the project. + db (Session): The database session, provided automatically. Returns: Response: The HTTP response object containing the downloaded file. @@ -943,6 +952,7 @@ async def download_task_boundaries( Args: project_id (int): The id of the project. + db (Session): The database session, provided automatically. Returns: Response: The HTTP response object containing the downloaded file. @@ -963,6 +973,7 @@ async def download_features(project_id: int, db: Session = Depends(database.get_ Args: project_id (int): The id of the project. + db (Session): The database session, provided automatically. Returns: Response: The HTTP response object containing the downloaded file. @@ -996,10 +1007,12 @@ async def generate_project_tiles( """Returns basemap tiles for a project. Args: + background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically. project_id (int): ID of project to create tiles for. source (str): Tile source ("esri", "bing", "topo", "google", "oam"). format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3". tms (str, optional): Default None. Custom TMS provider URL. + db (Session): The database session, provided automatically. Returns: str: Success message that tile generation started. @@ -1032,6 +1045,7 @@ async def tiles_list(project_id: int, db: Session = Depends(database.get_db)): Parameters: project_id: int + db (Session): The database session, provided automatically. Returns: Response: List of generated tiles for a project. @@ -1041,6 +1055,7 @@ async def tiles_list(project_id: int, db: Session = Depends(database.get_db)): @router.get("/download_tiles/") async def download_tiles(tile_id: int, db: Session = Depends(database.get_db)): + """Download the basemap tile archive for a project.""" log.debug("Getting tile archive path from DB") tiles_path = ( db.query(db_models.DbTilesPath) @@ -1071,6 +1086,7 @@ async def download_task_boundary_osm( Args: project_id (int): The id of the project. + db (Session): The database session, provided automatically. Returns: Response: The HTTP response object containing the downloaded file. @@ -1090,9 +1106,6 @@ async def download_task_boundary_osm( return response -from sqlalchemy.sql import text - - @router.get("/centroid/") async def project_centroid( project_id: int = None, @@ -1102,12 +1115,16 @@ async def project_centroid( Parameters: project_id (int): The ID of the project. + db (Session): The database session, provided automatically. Returns: - list[tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string. + list[tuple[int, str]]: A list of tuples containing the task ID and + the centroid as a string. """ query = text( - f"""SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_Centroid(outline)), ST_Y(ST_Centroid(outline))]) AS centroid + f"""SELECT id, + ARRAY_AGG(ARRAY[ST_X(ST_Centroid(outline)), + ST_Y(ST_Centroid(outline))]) AS centroid FROM projects WHERE {f"id={project_id}" if project_id else "1=1"} GROUP BY id;""" @@ -1136,9 +1153,6 @@ async def get_task_status( ) -from ..static import data_path - - @router.get("/templates/") async def get_template_file( file_type: str = Query( @@ -1174,6 +1188,7 @@ async def project_dashboard( Args: project_id (int): The ID of the project. + background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically. db (Session): The database session. Returns: @@ -1196,6 +1211,7 @@ async def get_contributors(project_id: int, db: Session = Depends(database.get_d Args: project_id (int): ID of project. + db (Session): The database session. Returns: list[project_schemas.ProjectUser]: List of project users. diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 9f81b34d4f..9ee400627e 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Pydantic schemas for Projects.""" import uuid from datetime import datetime @@ -32,24 +33,32 @@ class ODKCentral(BaseModel): + """ODK Central credentials.""" + odk_central_url: str odk_central_user: str odk_central_password: str class ProjectInfo(BaseModel): + """Basic project info.""" + name: str short_description: str description: str class ProjectUpdate(BaseModel): + """Update project.""" + name: Optional[str] = None short_description: Optional[str] = None description: Optional[str] = None class ProjectUpload(BaseModel): + """Upload new project.""" + author: User project_info: ProjectInfo xform_title: Optional[str] @@ -66,11 +75,15 @@ class ProjectUpload(BaseModel): class Feature(BaseModel): + """Features used for Task definitions.""" + id: int geometry: Optional[GeojsonFeature] = None class ProjectSummary(BaseModel): + """Project summaries.""" + id: int = -1 priority: ProjectPriority = ProjectPriority.MEDIUM priority_str: str = priority.name @@ -91,6 +104,7 @@ def from_db_project( cls, project: db_models.DbProject, ) -> "ProjectSummary": + """Generate model from database obj.""" priority = project.priority return cls( id=project.id, @@ -111,22 +125,28 @@ def from_db_project( class PaginationInfo(BaseModel): - hasNext: bool - hasPrev: bool - nextNum: Optional[int] + """Pagination JSON return.""" + + has_next: bool + has_prev: bool + next_num: Optional[int] page: int pages: int - prevNum: Optional[int] - perPage: int + prev_num: Optional[int] + per_page: int total: int class PaginatedProjectSummaries(BaseModel): + """Project summaries + Pagination info.""" + results: List[ProjectSummary] pagination: PaginationInfo class ProjectBase(BaseModel): + """Base project model.""" + id: int odkid: int author: User @@ -141,20 +161,28 @@ class ProjectBase(BaseModel): class ProjectOut(ProjectBase): + """Project display to user.""" + project_uuid: uuid.UUID = uuid.uuid4() class ReadProject(ProjectBase): + """Redundant model for refactor.""" + project_uuid: uuid.UUID = uuid.uuid4() location_str: Optional[str] = None class BackgroundTaskStatus(BaseModel): + """Background task status for project related tasks.""" + status: str message: Optional[str] = None class ProjectDashboard(BaseModel): + """Project details dashboard.""" + project_name_prefix: str organization: str total_tasks: int @@ -166,6 +194,7 @@ class ProjectDashboard(BaseModel): @field_serializer("last_active") def get_last_active(self, value, values): + """Date of last activity on project.""" if value is None: return None diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index 44854a46df..0e3acd76e9 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -113,7 +113,7 @@ def get_obj_from_bucket(bucket_name: str, s3_path: str) -> BytesIO: response = client.get_object(bucket_name, s3_path) return BytesIO(response.read()) except Exception as e: - raise ValueError(str(e)) + raise ValueError(str(e)) from e finally: if response: response.close() diff --git a/src/backend/app/submission/submission_crud.py b/src/backend/app/submission/submission_crud.py index 46ec181fb0..d4659a0764 100644 --- a/src/backend/app/submission/submission_crud.py +++ b/src/backend/app/submission/submission_crud.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Functions for task submissions.""" + import concurrent.futures import csv import io @@ -22,18 +24,16 @@ import os import threading import uuid -from asyncio import gather from collections import Counter from datetime import datetime, timedelta from io import BytesIO -from pathlib import Path import sozipfile.sozipfile as zipfile from asgiref.sync import async_to_sync from fastapi import HTTPException, Response from fastapi.responses import FileResponse from loguru import logger as log -from osm_fieldwork.json2osm import JsonDump +from osm_fieldwork.json2osm import json2osm from sqlalchemy.orm import Session from app.central.central_crud import get_odk_form, get_odk_project, list_odk_xforms @@ -45,8 +45,9 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None): """Gets the submission of project. - This function takes project_id and task_id as a parameter. - If task_id is provided, it returns all the submission made to that particular task, else all the submission made in the projects are returned. + + If task_id is provided, it submissions for a specific task, + else all the submission made for a project are returned. """ get_project_sync = async_to_sync(project_crud.get_project) project_info = get_project_sync(db, project_id) @@ -106,156 +107,19 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None) return submission_list -async def get_forms_of_project(db: Session, project_id: int): - project_info = await project_crud.get_project_by_id(db, project_id) - - # Return empty list if project is not found - if not project_info: - return [] - - odkid = project_info.odkid - project = get_odk_project() - - result = project.listForms(odkid) - return result - - -async def list_app_users_or_project(db: Session, project_id: int): - project_info = await project_crud.get_project_by_id(db, project_id) - - # Return empty list if project is not found - if not project_info: - return [] - - odkid = project_info.odkid - project = get_odk_project() - result = project.listAppUsers(odkid) - return result - - -# async def convert_json_to_osm_xml(file_path): - -# jsonin = JsonDump() -# infile = Path(file_path) - -# base = os.path.splitext(infile.name)[0] - -# osmoutfile = f"/tmp/{base}.osm" -# jsonin.createOSM(osmoutfile) - -# data = jsonin.parse(infile.as_posix()) - -# for entry in data: -# feature = jsonin.createEntry(entry) -# # Sometimes bad entries, usually from debugging XForm design, sneak in -# if len(feature) == 0: -# continue -# if len(feature) > 0: -# if "lat" not in feature["attrs"]: -# if 'geometry' in feature['tags']: -# if type(feature['tags']['geometry']) == str: -# coords = list(feature['tags']['geometry']) -# else: -# coords = feature['tags']['geometry']['coordinates'] -# feature['attrs'] = {'lat': coords[1], 'lon': coords[0]} -# else: -# log.warning("Bad record! %r" % feature) -# continue -# jsonin.writeOSM(feature) - -# jsonin.finishOSM() -# log.info("Wrote OSM XML file: %r" % osmoutfile) -# return osmoutfile - - -async def convert_json_to_osm_xml(file_path): - # TODO refactor to simply use json2osm(file_path) - jsonin = JsonDump() - infile = Path(file_path) - - base = os.path.splitext(infile.name)[0] - - osmoutfile = f"/tmp/{base}.osm" - jsonin.createOSM(osmoutfile) - - data = jsonin.parse(infile.as_posix()) - - async def process_entry_async(entry): - feature = jsonin.createEntry(entry) - if len(feature) == 0: - return None - if len(feature) > 0: - if "lat" not in feature["attrs"]: - if "geometry" in feature["tags"]: - if type(feature["tags"]["geometry"]) == str: - coords = list(feature["tags"]["geometry"]) - else: - coords = feature["tags"]["geometry"]["coordinates"] - feature["attrs"] = {"lat": coords[1], "lon": coords[0]} - else: - log.warning("Bad record! %r" % feature) - return None - return feature - - async def write_osm_async(features): - for feature in features: - if feature: - jsonin.writeOSM(feature) - jsonin.finishOSM() - log.info("Wrote OSM XML file: %r" % osmoutfile) - return osmoutfile - - data_processing_tasks = [process_entry_async(entry) for entry in data] - processed_features = await gather(*data_processing_tasks) - await write_osm_async(processed_features) - - return osmoutfile - - async def convert_json_to_osm(file_path): - # TODO refactor to simply use json2osm(file_path) - jsonin = JsonDump() - infile = Path(file_path) - - base = os.path.splitext(infile.name)[0] - - osmoutfile = f"/tmp/{base}.osm" - jsonin.createOSM(osmoutfile) - - jsonoutfile = f"/tmp/{base}.geojson" - jsonin.createGeoJson(jsonoutfile) - - data = jsonin.parse(infile.as_posix()) - - for entry in data: - feature = jsonin.createEntry(entry) - # Sometimes bad entries, usually from debugging XForm design, sneak in - if len(feature) == 0: - continue - if len(feature) > 0: - if "lat" not in feature["attrs"]: - if "geometry" in feature["tags"]: - if type(feature["tags"]["geometry"]) == str: - coords = list(feature["tags"]["geometry"]) - # del feature['tags']['geometry'] - else: - coords = feature["tags"]["geometry"]["coordinates"] - # del feature['tags']['geometry'] - feature["attrs"] = {"lat": coords[1], "lon": coords[0]} - else: - log.warning("Bad record! %r" % feature) - continue - jsonin.writeOSM(feature) - jsonin.writeGeoJson(feature) + """Wrapper for osm-fieldwork json2osm. - jsonin.finishOSM() - jsonin.finishGeoJson() - log.info("Wrote OSM XML file: %r" % osmoutfile) - log.info("Wrote GeoJson file: %r" % jsonoutfile) - return osmoutfile, jsonoutfile + FIXME add json output to osm2json (in addition to default OSM XML output) + """ + # TODO check speed of json2osm + # TODO if slow response, use run_in_threadpool + osm_xml_path = json2osm(file_path) + return osm_xml_path async def convert_to_osm_for_task(odk_id: int, form_id: int, xform: any): + """Convert JSON --> OSM XML for a specific XForm/Task.""" # This file stores the submission data. file_path = f"/tmp/{odk_id}_{form_id}.json" @@ -268,12 +132,12 @@ async def convert_to_osm_for_task(odk_id: int, form_id: int, xform: any): with open(file_path, "wb") as f: f.write(file) - convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) - osmoutfile, jsonoutfile = convert_json_to_osm_sync(file_path) - return osmoutfile, jsonoutfile + osmoutfile = await convert_json_to_osm(file_path) + return osmoutfile def convert_to_osm(db: Session, project_id: int, task_id: int): + """Convert submissions to OSM XML format.""" get_project_sync = async_to_sync(project_crud.get_project) project_info = get_project_sync(db, project_id) @@ -321,9 +185,10 @@ def convert_to_osm(db: Session, project_id: int, task_id: int): # Convert the submission to osm xml format convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) - osmoutfile, jsonoutfile = convert_json_to_osm_sync(jsoninfile) + osmoutfile = convert_json_to_osm_sync(jsoninfile) - if osmoutfile and jsonoutfile: + # if osmoutfile and jsonoutfile: + if osmoutfile: # FIXME: Need to fix this when generating osm file # Remove the extra closing tag from the end of the file @@ -344,7 +209,7 @@ def convert_to_osm(db: Session, project_id: int, task_id: int): # Add the files to the ZIP file with zipfile.ZipFile(final_zip_file_path, mode="a") as final_zip_file: final_zip_file.write(osmoutfile) - final_zip_file.write(jsonoutfile) + # final_zip_file.write(jsonoutfile) return FileResponse(final_zip_file_path) @@ -380,7 +245,8 @@ def gather_all_submission_csvs(db, project_id): def download_submission_for_task(task_id): log.info( - f"Thread {threading.current_thread().name} - Downloading submission for Task ID {task_id}" + f"Thread {threading.current_thread().name} - " + f"Downloading submission for Task ID {task_id}" ) xml_form_id = f"{project_name}_{form_category}_{task_id}".split("_")[2] file = xform.getSubmissionMedia(odkid, xml_form_id) @@ -391,7 +257,8 @@ def download_submission_for_task(task_id): def extract_files(zip_file_path): log.info( - f"Thread {threading.current_thread().name} - Extracting files from {zip_file_path}" + f"Thread {threading.current_thread().name} - " + f"Extracting files from {zip_file_path}" ) with zipfile.ZipFile(zip_file_path, "r") as zip_file: extract_dir = os.path.splitext(zip_file_path)[0] @@ -414,11 +281,14 @@ def extract_files(zip_file_path): file_path = future.result() files.append(file_path) log.info( - f"Thread {threading.current_thread().name} - Task {task_id} - Download completed." + f"Thread {threading.current_thread().name} -" + f" Task {task_id} - Download completed." ) except Exception as e: log.error( - f"Thread {threading.current_thread().name} - Error occurred while downloading submission for task {task_id}: {e}" + f"Thread {threading.current_thread().name} -" + f" Error occurred while downloading submission for task " + f"{task_id}: {e}" ) # Extract files using thread pool @@ -431,11 +301,13 @@ def extract_files(zip_file_path): try: extracted_files.extend(future.result()) log.info( - f"Thread {threading.current_thread().name} - Extracted files from {file_path}" + f"Thread {threading.current_thread().name} -" + f" Extracted files from {file_path}" ) except Exception as e: log.error( - f"Thread {threading.current_thread().name} - Error occurred while extracting files from {file_path}: {e}" + f"Thread {threading.current_thread().name} -" + f" Error occurred while extracting files from {file_path}: {e}" ) # Create a new ZIP file for the extracted files @@ -450,6 +322,7 @@ def extract_files(zip_file_path): def update_submission_in_s3( db: Session, project_id: int, background_task_id: uuid.UUID ): + """Update or create new submission JSON in S3 for a project.""" try: # Get Project get_project_sync = async_to_sync(project_crud.get_project) @@ -567,49 +440,52 @@ def get_all_submissions_json(db: Session, project_id): return submissions -def get_project_submission(db: Session, project_id: int): - get_project_sync = async_to_sync(project_crud.get_project) - project_info = get_project_sync(db, project_id) +# TODO delete me +# def get_project_submission(db: Session, project_id: int): +# """Get.""" +# get_project_sync = async_to_sync(project_crud.get_project) +# project_info = get_project_sync(db, project_id) - # Return empty list if project is not found - if not project_info: - raise HTTPException(status_code=404, detail="Project not found") +# # Return empty list if project is not found +# if not project_info: +# raise HTTPException(status_code=404, detail="Project not found") - odkid = project_info.odkid - project_name = project_info.project_name_prefix - form_category = project_info.xform_title - project_tasks = project_info.tasks +# odkid = project_info.odkid +# project_name = project_info.project_name_prefix +# form_category = project_info.xform_title +# project_tasks = project_info.tasks - # ODK Credentials - odk_credentials = project_schemas.ODKCentral( - odk_central_url=project_info.odk_central_url, - odk_central_user=project_info.odk_central_user, - odk_central_password=project_info.odk_central_password, - ) +# # ODK Credentials +# odk_credentials = project_schemas.ODKCentral( +# odk_central_url=project_info.odk_central_url, +# odk_central_user=project_info.odk_central_user, +# odk_central_password=project_info.odk_central_password, +# ) - # Get ODK Form with odk credentials from the project. - xform = get_odk_form(odk_credentials) +# # Get ODK Form with odk credentials from the project. +# xform = get_odk_form(odk_credentials) - submissions = [] +# submissions = [] - task_list = [x.id for x in project_tasks] - for id in task_list: - xml_form_id = f"{project_name}_{form_category}_{id}".split("_")[2] - file = xform.getSubmissions(odkid, xml_form_id, None, False, True) - if not file: - json_data = None - else: - json_data = json.loads(file) - json_data_value = json_data.get("value") - if json_data_value: - submissions.extend(json_data_value) +# task_list = [x.id for x in project_tasks] +# for id in task_list: +# xml_form_id = f"{project_name}_{form_category}_{id}".split("_")[2] +# file = xform.getSubmissions(odkid, xml_form_id, None, False, True) +# if not file: +# json_data = None +# else: +# json_data = json.loads(file) +# json_data_value = json_data.get("value") +# if json_data_value: +# submissions.extend(json_data_value) - return submissions +# return submissions async def download_submission( db: Session, project_id: int, task_id: int, export_json: bool ): + """Download submission data from ODK Central and aggregate.""" project_info = await project_crud.get_project(db, project_id) # Return empty list if project is not found @@ -639,7 +515,8 @@ async def download_submission( task_list = [x.id for x in project_tasks] - # zip_file_path = f"{project_name}_{form_category}_submissions.zip" # Create a new ZIP file for all submissions + # # Create a new ZIP file for all submissions + # zip_file_path = f"{project_name}_{form_category}_submissions.zip" files = [] for id in task_list: @@ -653,16 +530,15 @@ async def download_submission( with open(file_path, "wb") as f: f.write(file.content) - files.append( - file_path - ) # Add the output file path to the list of files for the final ZIP file + # Add the output file path to the list of files for the final ZIP file + files.append(file_path) extracted_files = [] for file_path in files: with zipfile.ZipFile(file_path, "r") as zip_file: - zip_file.extractall( - os.path.splitext(file_path)[0] - ) # Extract the contents of the nested ZIP files to a directory with the same name as the ZIP file + # Extract the contents of the nested ZIP files to a directory + # with the same name as the ZIP file + zip_file.extractall(os.path.splitext(file_path)[0]) extracted_files += [ os.path.join(os.path.splitext(file_path)[0], f) for f in zip_file.namelist() @@ -717,9 +593,9 @@ async def download_submission( async def get_submission_points(db: Session, project_id: int, task_id: int = None): """Gets the submission points of project. - This function takes project_id and task_id as a parameter. - If task_id is provided, it returns all the submission points made to that particular task, - else all the submission points made in the projects are returned. + + If task_id is provided, it return point specific to a task, + else the entire project. """ project_info = await project_crud.get_project_by_id(db, project_id) @@ -760,8 +636,10 @@ async def get_submission_points(db: Session, project_id: int, task_id: int = Non csv_reader = csv.DictReader(io.TextIOWrapper(csv_file)) geometry = [] for row in csv_reader: - # Check if the row contains the 'warmup-Latitude' and 'warmup-Longitude' columns - # FIXME: fix the column names (they might not be same warmup-Latitude and warmup-Longitude) + # Check if the row contains the 'warmup-Latitude' and + # 'warmup-Longitude' columns + # FIXME: fix the column names (they might not be same + # warmup-Latitude and warmup-Longitude) if "warmup-Latitude" in row and "warmup-Longitude" in row: point = (row["warmup-Latitude"], row["warmup-Longitude"]) @@ -782,6 +660,7 @@ async def get_submission_points(db: Session, project_id: int, task_id: int = Non async def get_submission_count_of_a_project(db: Session, project_id: int): + """Return the total number of submissions made for a project.""" project_info = await project_crud.get_project(db, project_id) # Return empty list if project is not found @@ -831,6 +710,7 @@ async def get_submissions_by_date( db (Session): The database session. project_id (int): The ID of the project. days (int): The number of days to consider for fetching submissions. + planned_task (int): Associated task id. Returns: dict: A dictionary containing the submission counts for each date. diff --git a/src/backend/app/submission/submission_routes.py b/src/backend/app/submission/submission_routes.py index 2046b62eb3..24771d5445 100644 --- a/src/backend/app/submission/submission_routes.py +++ b/src/backend/app/submission/submission_routes.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Routes associated with data submission to and from ODK Central.""" + import json import os from typing import Optional @@ -46,52 +48,21 @@ async def read_submissions( project_id: int, task_id: int = None, db: Session = Depends(database.get_db), -): - """This api returns the submission made in the project. - It takes two parameters: project_id and task_id. - +) -> list[dict]: + """Get all submissions made for a project. - project_id: The ID of the project. This endpoint returns the submission made in this project. - - task_id: The ID of the task. This parameter is optional. If task_id is provided, this endpoint returns the submissions made for this task. + Args: + project_id (int): The ID of the project. + task_id (int, optional): The ID of the task. + If provided, returns the submissions made for a specific task only. + db (Session): The database session, automatically provided. - Returns the list of submissions. + Returns: + list[dict]: The list of submissions. """ return submission_crud.get_submission_of_project(db, project_id, task_id) -@router.get("/list-forms") -async def list_forms( - project_id: int, - db: Session = Depends(database.get_db), -): - """This api returns the list of forms in the odk central. - - It takes one parameter: project_id. - - project_id: The ID of the project. This endpoint returns the list of forms in this project. - - Returns the list of forms details provided by the central api. - """ - return await submission_crud.get_forms_of_project(db, project_id) - - -@router.get("/list-app-users") -async def list_app_users( - project_id: int, - db: Session = Depends(database.get_db), -): - """This api returns the list of forms in the odk central. - - It takes one parameter: project_id. - - project_id: The ID of the project. This endpoint returns the list of forms in this project. - - Returns the list of forms details provided by the central api. - """ - return await submission_crud.list_app_users_or_project(db, project_id) - - @router.get("/download") async def download_submission( project_id: int, @@ -99,13 +70,19 @@ async def download_submission( export_json: bool = True, db: Session = Depends(database.get_db), ): - """This api downloads the the submission made in the project. - It takes two parameters: project_id and task_id. + """Download the submissions for a given project. - project_id: The ID of the project. This endpoint returns the submission made in this project. + Returned as either a JSONResponse, or a file to download. - task_id: The ID of the task. This parameter is optional. If task_id is provided, this endpoint returns the submissions made for this task. + Args: + project_id (int): The ID of the project. + task_id (int, optional): The ID of the task. + If provided, returns the submissions made for a specific task only. + export_json (bool): Export in JSON format, else returns a file. + db (Session): The database session, automatically provided. + Returns: + Union[list[dict], File]: JSON of submissions, or submission file. """ if not (task_id or export_json): file = submission_crud.gather_all_submission_csvs(db, project_id) @@ -122,11 +99,16 @@ async def submission_points( task_id: int = None, db: Session = Depends(database.get_db), ): - """This api returns the submission points of a project. - It takes two parameter: project_id and task_id. + """Get submission points for a given project. - project_id: The ID of the project. This endpoint returns the submission points of this project. - task_id: The task_id of the project. This endpoint returns the submission points of this task. + Args: + project_id (int): The ID of the project. + task_id (int, optional): The ID of the task. + If provided, returns the submissions made for a specific task only. + db (Session): The database session, automatically provided. + + Returns: + File: a zip containing submission points. """ return await submission_crud.get_submission_points(db, project_id, task_id) @@ -136,14 +118,17 @@ async def convert_to_osm( project_id: int, task_id: int = None, db: Session = Depends(database.get_db), -): - """This api converts the submission to osm format. - It takes two parameter: project_id and task_id. +) -> str: + """Convert JSON submissions to OSM XML for a project. - task_id is optional. - If task_id is provided, this endpoint converts the submission of this task. - If task_id is not provided, this endpoint converts the submission of the whole project. + Args: + project_id (int): The ID of the project. + task_id (int, optional): The ID of the task. + If provided, returns the submissions made for a specific task only. + db (Session): The database session, automatically provided. + Returns: + File: an OSM XML of submissions. """ # NOTE runs in separate thread using run_in_threadpool converted = await run_in_threadpool( @@ -157,6 +142,7 @@ async def get_submission_count( project_id: int, db: Session = Depends(database.get_db), ): + """Get the submission count for a project.""" return await submission_crud.get_submission_count_of_a_project(db, project_id) @@ -165,6 +151,7 @@ async def conflate_osm_data( project_id: int, db: Session = Depends(database.get_db), ): + """Conflate submission data against existing OSM data.""" # All Submissions JSON # NOTE runs in separate thread using run_in_threadpool submission = await run_in_threadpool( @@ -192,7 +179,7 @@ async def conflate_osm_data( f.write(json.dumps(submission)) # Convert the submission to osm xml format - osmoutfile, jsonoutfile = await submission_crud.convert_json_to_osm(jsoninfile) + osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) # Remove the extra closing tag from the end of the file with open(osmoutfile, "r") as f: @@ -225,6 +212,10 @@ async def download_submission_json( background_task_id: Optional[str] = None, db: Session = Depends(database.get_db), ): + """Download submissions for a project in JSON format. + + TODO check for redundancy with submission/download endpoint and refactor. + """ # Get Project project = await project_crud.get_project(db, project_id) @@ -268,6 +259,10 @@ async def get_osm_xml( project_id: int, db: Session = Depends(database.get_db), ): + """Get the submissions in OSM XML format for a project. + + TODO refactor to put logic in crud for easier testing. + """ # JSON FILE PATH jsoninfile = f"/tmp/{project_id}_json_infile.json" @@ -286,7 +281,7 @@ async def get_osm_xml( f.write(json.dumps(submission)) # Convert the submission to osm xml format - osmoutfile = await submission_crud.convert_json_to_osm_xml(jsoninfile) + osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) # Remove the extra closing tag from the end of the file with open(osmoutfile, "r") as f: @@ -316,9 +311,17 @@ async def get_submission_page( planned_task: Optional[int] = None, db: Session = Depends(database.get_db), ): - """This api returns the submission page of a project. - It takes one parameter: project_id. - project_id: The ID of the project. This endpoint returns the submission page of this project. + """Summary submissison details for submission page. + + Args: + background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically. + db (Session): The database session, automatically generated. + project_id (int): The ID of the project. + days (int): The number of days to consider for fetching submissions. + planned_task (int): Associated task id. + + Returns: + dict: A dictionary containing the submission counts for each date. """ data = await submission_crud.get_submissions_by_date( db, project_id, days, planned_task @@ -344,7 +347,7 @@ async def get_submission_form_fields( Args: project_id (int): The ID of the project. - db (Session, optional): The database session. Defaults to Depends(database.get_db). + db (Session): The database session, automatically generated. Returns: Any: The response from the submission form API. diff --git a/src/backend/app/submission/submission_schemas.py b/src/backend/app/submission/submission_schemas.py index 721ae2a66d..b2b30015e2 100644 --- a/src/backend/app/submission/submission_schemas.py +++ b/src/backend/app/submission/submission_schemas.py @@ -15,3 +15,5 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # + +"""Pydantic models for data submissions.""" diff --git a/src/backend/app/tasks/tasks_crud.py b/src/backend/app/tasks/tasks_crud.py index 2671820710..3b24cd4d44 100644 --- a/src/backend/app/tasks/tasks_crud.py +++ b/src/backend/app/tasks/tasks_crud.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Logic for FMTM tasks.""" + import base64 from typing import List @@ -40,6 +42,7 @@ async def get_task_count_in_project(db: Session, project_id: int): + """Get task count for a project.""" query = text(f"""select count(*) from tasks where project_id = {project_id}""") result = db.execute(query) return result.fetchone()[0] @@ -66,6 +69,7 @@ async def get_task_id_list(db: Session, project_id: int) -> list[int]: async def get_tasks( db: Session, project_id: int, user_id: int, skip: int = 0, limit: int = 1000 ): + """Get task details for a project.""" if project_id: db_tasks = ( db.query(db_models.DbTask) @@ -88,12 +92,15 @@ async def get_tasks( async def get_task(db: Session, task_id: int): + """Get details for a specific task ID.""" + log.debug(f"Getting task with ID '{task_id}' from database") return db.query(db_models.DbTask).filter(db_models.DbTask.id == task_id).first() async def update_task_status( db: Session, user_id: int, task_id: int, new_status: TaskStatus ): + """Update the status of a task.""" log.debug(f"Updating task ID {task_id} to status {new_status}") if not user_id: log.error(f"User id is not present: {user_id}") @@ -163,7 +170,10 @@ async def update_task_status( else: raise HTTPException( status_code=400, - detail=f"Not a valid status update: {db_task.task_status.name} to {new_status.name}", + detail=( + f"Not a valid status update: " + f"{db_task.task_status.name} to {new_status.name}" + ), ) @@ -175,7 +185,11 @@ async def update_task_status( async def create_task_history_for_status_change( db_task: db_models.DbTask, new_status: TaskStatus, db_user: db_models.DbUser ): - msg = f"Status changed from {db_task.task_status.name} to {new_status.name} by: {db_user.username}" + """Append task status change to task history.""" + msg = ( + f"Status changed from {db_task.task_status.name} " + f"to {new_status.name} by: {db_user.username}" + ) log.info(msg) new_task_history = db_models.DbTaskHistory( @@ -211,6 +225,7 @@ async def get_qr_codes_for_task( db: Session, task_id: int, ): + """Get the ODK Collect QR code for a task area.""" task = await get_task(db=db, task_id=task_id) if task: if task.qr_code: @@ -224,12 +239,6 @@ async def get_qr_codes_for_task( raise HTTPException(status_code=400, detail="Task does not exist") -async def get_task_by_id(db: Session, task_id: int): - task = db.query(db_models.DbTask).filter(db_models.DbTask.id == task_id).first() - print("Task ", task) - return task - - async def update_task_files( db: Session, project_id: int, @@ -239,6 +248,7 @@ async def update_task_files( category: str, task_boundary: str, ): + """Update associated files for a task.""" # This file will store osm extracts task_polygons = f"/tmp/{project_name}_{category}_{task_id}.geojson" @@ -268,7 +278,8 @@ async def update_task_files( # Collect feature mappings for bulk insert for feature in outline_geojson["features"]: - # If the osm extracts contents do not have a title, provide an empty text for that. + # If the osm extracts contents do not have a title, + # provide an empty text for that feature["properties"]["title"] = "" feature_shape = shape(feature["geometry"]) @@ -284,7 +295,8 @@ async def update_task_files( db.add(db_feature) db.commit() - # Update task_polygons file containing osm extracts with the new geojson contents containing title in the properties. + # Update task_polygons file containing osm extracts with the new + # geojson contents containing title in the properties. with open(task_polygons, "w") as jsonfile: jsonfile.truncate(0) # clear the contents of the file dump(updated_outline_geojson, jsonfile) @@ -300,7 +312,7 @@ async def edit_task_boundary(db: Session, task_id: int, boundary: str): geometry = boundary["features"][0]["geometry"] outline = shape(geometry) - task = await get_task_by_id(db, task_id) + task = await get_task(db, task_id) if not task: raise HTTPException(status_code=404, detail="Task not found") @@ -324,6 +336,8 @@ async def edit_task_boundary(db: Session, task_id: int, boundary: str): async def update_task_history( tasks: List[tasks_schemas.TaskBase], db: Session = Depends(database.get_db) ): + """Update task history with username and user profile image.""" + def process_history_entry(history_entry): status = history_entry.action_text.split() history_entry.status = status[5] diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index 6df7163a47..8e5d0b3d3f 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Routes for FMTM tasks.""" import json from typing import List @@ -27,10 +28,9 @@ from app.db import database from app.models.enums import TaskStatus from app.projects import project_crud, project_schemas +from app.tasks import tasks_crud, tasks_schemas from app.users import user_schemas -from . import tasks_crud, tasks_schemas - router = APIRouter( prefix="/tasks", tags=["tasks"], @@ -45,6 +45,7 @@ async def read_task_list( limit: int = 1000, db: Session = Depends(database.get_db), ): + """Get the task list for a project.""" tasks = await tasks_crud.get_tasks(db, project_id, limit) updated_tasks = await tasks_crud.update_task_history(tasks, db) if not tasks: @@ -60,6 +61,7 @@ async def read_tasks( limit: int = 1000, db: Session = Depends(database.get_db), ): + """Get all task details, either for a project or user.""" if user_id: raise HTTPException( status_code=300, @@ -80,11 +82,14 @@ async def get_point_on_surface(project_id: int, db: Session = Depends(database.g project_id (int): The ID of the project. Returns: - List[Tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string. + List[Tuple[int, str]]: A list of tuples containing the task ID + and the centroid as a string. """ query = text( f""" - SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_PointOnSurface(outline)), ST_Y(ST_PointOnSurface(outline))]) AS point + SELECT id, + ARRAY_AGG(ARRAY[ST_X(ST_PointOnSurface(outline)), + ST_Y(ST_PointOnSurface(outline))]) AS point FROM tasks WHERE project_id = {project_id} GROUP BY id; """ @@ -104,7 +109,8 @@ async def get_tasks_near_me( @router.get("/{task_id}", response_model=tasks_schemas.Task) -async def read_tasks(task_id: int, db: Session = Depends(database.get_db)): +async def get_specific_task(task_id: int, db: Session = Depends(database.get_db)): + """Get a specific task by it's ID.""" task = await tasks_crud.get_task(db, task_id) if not task: raise HTTPException(status_code=404, detail="Task not found") @@ -120,7 +126,7 @@ async def update_task_status( new_status: TaskStatus, db: Session = Depends(database.get_db), ): - # TODO verify logged in user + """Update the task status.""" user_id = user.id task = await tasks_crud.update_task_status(db, user_id, task_id, new_status) @@ -135,6 +141,7 @@ async def get_qr_code_list( task_id: int, db: Session = Depends(database.get_db), ): + """Get the associated ODK Collect QR code for a task.""" return await tasks_crud.get_qr_codes_for_task(db=db, task_id=task_id) @@ -144,6 +151,7 @@ async def edit_task_boundary( boundary: UploadFile = File(...), db: Session = Depends(database.get_db), ): + """Update the task boundary manually.""" # read entire file content = await boundary.read() boundary_json = json.loads(content) @@ -158,6 +166,7 @@ async def task_features_count( project_id: int, db: Session = Depends(database.get_db), ): + """Get all features within a task area.""" # Get the project object. project = await project_crud.get_project(db, project_id) @@ -175,7 +184,8 @@ async def task_features_count( for x in odk_details: feature_count_query = text( f""" - select count(*) from features where project_id = {project_id} and task_id = {x['xmlFormId']} + select count(*) from features + where project_id = {project_id} and task_id = {x['xmlFormId']} """ ) diff --git a/src/backend/app/users/user_crud.py b/src/backend/app/users/user_crud.py index 20d5a420b3..922d2508c9 100644 --- a/src/backend/app/users/user_crud.py +++ b/src/backend/app/users/user_crud.py @@ -20,8 +20,8 @@ from sqlalchemy.orm import Session -from ..db import db_models -from . import user_schemas +from app.db import db_models +from app.users import user_schemas # -------------- # ---- CRUD ---- diff --git a/src/backend/app/users/user_routes.py b/src/backend/app/users/user_routes.py index 3b6f0d4d15..085c49e15d 100644 --- a/src/backend/app/users/user_routes.py +++ b/src/backend/app/users/user_routes.py @@ -22,9 +22,9 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.orm import Session -from ..db import database -from ..models.enums import UserRole as UserRoleEnum -from . import user_crud, user_schemas +from app.db import database +from app.models.enums import UserRole as UserRoleEnum +from app.users import user_crud, user_schemas router = APIRouter( prefix="/users", diff --git a/src/backend/tests/__init__.py b/src/backend/tests/__init__.py index e69de29bb2..5581e5c3a1 100644 --- a/src/backend/tests/__init__.py +++ b/src/backend/tests/__init__.py @@ -0,0 +1 @@ +"""Backend tests using PyTest."""