From c203b4a87d22c2ab7f01c820630e87888c7fa489 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 8 Jan 2024 10:06:18 +0000 Subject: [PATCH] refactor: allow generating basemaps by user --- src/backend/app/projects/project_crud.py | 132 ++++++--------------- src/backend/app/projects/project_routes.py | 44 ++++--- 2 files changed, 58 insertions(+), 118 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 62183116a8..d3705aff97 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -66,13 +66,17 @@ from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson, timestamp from app.organization import organization_crud from app.projects import project_schemas -from app.s3 import add_obj_to_bucket, get_obj_from_bucket +from app.s3 import ( + add_file_to_bucket, + add_obj_to_bucket, + get_bucket_path, + get_obj_from_bucket, +) from app.tasks import tasks_crud from app.users import user_crud QR_CODES_DIR = "QR_codes/" TASK_GEOJSON_DIR = "geojson/" -TILESDIR = "/opt/tiles" async def get_projects( @@ -2122,10 +2126,19 @@ def get_project_tiles( project_id: int, background_task_id: uuid.UUID, source: str, + +# NOTE defined as non-async to run in separate thread +def generate_basemap_for_bbox( + db: Session, + project_id: int, + bbox: tuple, + background_task_id: uuid.UUID, + source: str, output_format: str = "mbtiles", tms: str = None, + task_id: int = None, ): - """Get the tiles for a project. + """Get basemap tiles for a project. Args: db (Session): SQLAlchemy db session. @@ -2135,11 +2148,11 @@ def get_project_tiles( output_format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3". tms (str, optional): Default None. Custom TMS provider URL. + task_id (bool): If set, create for a task boundary only. """ zooms = "12-19" - tiles_path_id = uuid.uuid4() - tiles_dir = f"{TILESDIR}/{tiles_path_id}" - outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}" + tiles_dir = "opt/tiles" + outfile = f"/tmp/{project_id}_{uuid.uuid4()}.{output_format}" tile_path_instance = db_models.DbTilesPath( project_id=project_id, @@ -2153,36 +2166,9 @@ def get_project_tiles( db.add(tile_path_instance) db.commit() - # Project Outline - log.debug(f"Getting bbox for project: {project_id}") - query = text( - f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon, - ST_YMin(ST_Envelope(outline)) AS min_lat, - ST_XMax(ST_Envelope(outline)) AS max_lon, - ST_YMax(ST_Envelope(outline)) AS max_lat - FROM projects - WHERE id = {project_id};""" - ) - - result = db.execute(query) - project_bbox = result.fetchone() - log.debug(f"Extracted project bbox: {project_bbox}") + # Get coords from bbox + min_lon, min_lat, max_lon, max_lat = bbox - if project_bbox: - min_lon, min_lat, max_lon, max_lat = project_bbox - else: - log.error(f"Failed to get bbox from project: {project_id}") - - log.debug( - "Creating basemap with params: " - f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | " - f"outfile={outfile} | " - f"zooms={zooms} | " - f"outdir={tiles_dir} | " - f"source={source} | " - f"xy={False} | " - f"tms={tms}" - ) create_basemap_file( boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}", outfile=outfile, @@ -2194,77 +2180,24 @@ def get_project_tiles( ) log.info(f"Basemap created for project ID {project_id}: {outfile}") + get_bucket_path_sync = async_to_sync(get_bucket_path) + project_s3_path = get_bucket_path_sync(db, project_id) - get_project_sync = async_to_sync(get_project) - project = get_project_sync(db, project_id) + if task_id: + s3_tile_path = f"{project_s3_path}/basemaps/{task_id}.{output_format}" + else: + s3_tile_path = f"{project_s3_path}/basemap.{output_format}" - from app.s3 import add_file_to_bucket add_file_to_bucket( settings.S3_BUCKET_NAME, - f"/{project.organisation_id}/{project_id}/basemap.mbtiles", - outfile + s3_tile_path, + outfile, ) - # Generate mbtiles for each task - get_tasks_async = async_to_sync(tasks_crud.get_task_id_list) - task_list = get_tasks_async(db, project_id) - - for task_id in task_list: - try: - log.debug(f"Getting bbox for task: {task_id}") - query = text( - f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon, - ST_YMin(ST_Envelope(outline)) AS min_lat, - ST_XMax(ST_Envelope(outline)) AS max_lon, - ST_YMax(ST_Envelope(outline)) AS max_lat - FROM tasks - WHERE id = {task_id};""" - ) - - result = db.execute(query) - task_bbox = result.fetchone() - log.debug(f"Extracted task bbox: {task_bbox}") - - if task_bbox: - min_lon, min_lat, max_lon, max_lat = task_bbox - else: - log.error(f"Failed to get bbox from task: {project_id}") - - task_basemap_outfile = f"{tiles_dir}/{task_id}_{source}tiles.{output_format}" - - log.debug( - "Creating basemap with params: " - f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | " - f"outfile={task_basemap_outfile} | " - f"zooms={zooms} | " - f"outdir={tiles_dir} | " - f"source={source} | " - f"xy={False} | " - f"tms={tms}" - ) - create_basemap_file( - boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}", - outfile=task_basemap_outfile, - zooms=zooms, - outdir=tiles_dir, - source=source, - xy=False, - tms=tms, - ) - log.info(f"Basemap created for task ID {task_id}: {task_basemap_outfile}") - except Exception as e: - log.error(str(e)) - continue - - # Upload task mbtile to s3 - add_file_to_bucket( - settings.S3_BUCKET_NAME, - f"/{project.organisation_id}/{project_id}/basemap/{task_id}.mbtiles", - task_basemap_outfile - ) - - tile_path_instance.status = 4 + tile_path_instance.path = ( + f"{settings.S3_DOWNLOAD_ROOT}/" f"{settings.S3_BUCKET_NAME}{s3_tile_path}" + ) db.commit() # Update background task status to COMPLETED @@ -2278,6 +2211,7 @@ def get_project_tiles( log.error(f"Tiles generation process failed for project id {project_id}") tile_path_instance.status = 2 + tile_path_instance.path = "" db.commit() # Update background task status to FAILED diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 5c36e420fb..29c3f669db 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -20,7 +20,6 @@ import json import os import uuid -from pathlib import Path from typing import Optional import geojson @@ -35,21 +34,20 @@ Response, UploadFile, ) -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse, JSONResponse, RedirectResponse from loguru import logger as log from osm_fieldwork.make_data_extract import getChoices from osm_fieldwork.xlsforms import xlsforms_path from sqlalchemy.orm import Session from app.auth.osm import AuthUser, login_required +from app.central import central_crud +from app.db import database, db_models +from app.models.enums import TILES_FORMATS, TILES_SOURCE +from app.projects import project_crud, project_schemas +from app.projects.project_crud import check_crs from app.submission import submission_crud - -from ..central import central_crud -from ..db import database, db_models -from ..models.enums import TILES_FORMATS, TILES_SOURCE -from ..tasks import tasks_crud -from . import project_crud, project_schemas -from .project_crud import check_crs +from app.tasks import tasks_crud router = APIRouter( prefix="/projects", @@ -990,6 +988,10 @@ async def download_features(project_id: int, db: Session = Depends(database.get_ async def generate_project_tiles( background_tasks: BackgroundTasks, project_id: int, + task_id: str = Query( + None, + description="Optional task id to generate for", + ), source: str = Query( ..., description="Select a source for tiles", enum=TILES_SOURCE ), @@ -1006,12 +1008,13 @@ async def generate_project_tiles( Args: project_id (int): ID of project to create tiles for. + task_id (int): Optional task ID (task area) to generate for. source (str): Tile source ("esri", "bing", "topo", "google", "oam"). format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3". tms (str, optional): Default None. Custom TMS provider URL. Returns: - str: Success message that tile generation started. + dict: Success message that tile generation started. """ # Create task in db and return uuid log.debug( @@ -1022,18 +1025,19 @@ async def generate_project_tiles( db, name="generate tiles", project_id=project_id ) - background_tasks.add_task( - project_crud.get_project_tiles, + project_crud.generate_project_or_task_basemap, db, project_id, background_task_id, source, format, tms, + task_id, ) - return {"Message": "Tile generation started"} + return JSONResponse(status_code=200, content={"success": True}) + @router.get("/tiles_list/{project_id}/") @@ -1173,14 +1177,15 @@ async def get_template_file( ) -@router.get("/project_dashboard/{project_id}", response_model=project_schemas.ProjectDashboard) +@router.get( + "/project_dashboard/{project_id}", response_model=project_schemas.ProjectDashboard +) async def project_dashboard( - project_id: int, + project_id: int, background_tasks: BackgroundTasks, - db: Session = Depends(database.get_db) + db: Session = Depends(database.get_db), ): - """ - Get the project dashboard details. + """Get the project dashboard details. Args: project_id (int): The ID of the project. @@ -1199,6 +1204,7 @@ async def project_dashboard( ) return data + @router.get("/contributors/{project_id}") async def get_contributors(project_id: int, db: Session = Depends(database.get_db)): """Get contributors of a project. @@ -1210,4 +1216,4 @@ async def get_contributors(project_id: int, db: Session = Depends(database.get_d list[project_schemas.ProjectUser]: List of project users. """ project_users = await project_crud.get_project_users(db, project_id) - return project_users \ No newline at end of file + return project_users