feat(mapper): frontend pmtile basemap management pt2 (#1925)

* fix(backend): file upload methods in s3.py

* fix(backend): upload basemaps to s3, remove get/download endpoint

* fix(frontend): access basemap urls directly from s3

* feat(backend): add methods for deleting DbBasemap & DbBackgroundTask

* feat(backend): add manual migration script for basemap upload to s3

* build: move upload basemaps script to migration dir

* feat(mapper): improve offline pmtiles experience, load opfs by default

* refactor: remove console log on search for opfs basemap

* refactor: remove custom logic for loading offline pmtiles in react frontend

* fix(management): allow loading custom pmtiles basemaps on react frontend

* fix(mapper): automatically select pmtiles layer if set

src/backend/app/db/models.py

@@ -1523,6 +1523,23 @@ async def update(
 
         return updated_task
 
+    @classmethod
+    async def delete(cls, db: Connection, background_task_id: UUID) -> bool:
+        """Delete a background task entry."""
+        sql = """
+            DELETE from background_tasks
+            WHERE id = %(background_task_id)s
+            RETURNING id;
+        """
+
+        async with db.cursor() as cur:
+            await cur.execute(sql, {"background_task_id": background_task_id})
+            success = await cur.fetchone()
+
+        if success:
+            return True
+        return False
+
 
 class DbBasemap(BaseModel):
     """Table tiles_path.
@@ -1636,7 +1653,7 @@ async def create(
     async def update(
         cls,
         db: Connection,
-        basemap_id: int,
+        basemap_id: UUID,
         basemap_update: "BasemapUpdate",
     ) -> Self:
         """Update values for a basemap."""
@@ -1662,6 +1679,23 @@ async def update(
 
         return updated_basemap
 
+    @classmethod
+    async def delete(cls, db: Connection, basemap_id: UUID) -> bool:
+        """Delete a basemap."""
+        sql = """
+            DELETE from basemaps
+            WHERE id = %(basemap_id)s
+            RETURNING id;
+        """
+
+        async with db.cursor() as cur:
+            await cur.execute(sql, {"basemap_id": basemap_id})
+            success = await cur.fetchone()
+
+        if success:
+            return True
+        return False
+
 
 class DbSubmissionPhoto(BaseModel):
     """Table submission_photo.

src/backend/app/projects/project_crud.py

@@ -49,7 +49,7 @@
     split_geojson_by_task_areas,
 )
 from app.projects import project_deps, project_schemas
-from app.s3 import add_obj_to_bucket
+from app.s3 import add_file_to_bucket, add_obj_to_bucket
 
 TILESDIR = "/opt/tiles"
 
@@ -686,16 +686,20 @@ async def get_project_features_geojson(
 def generate_project_basemap(
     db: Connection,
     project_id: int,
+    org_id: int,
     background_task_id: uuid.UUID,
     source: str,
     output_format: str = "mbtiles",
     tms: Optional[str] = None,
 ):
     """Get the tiles for a project.
 
+    FIXME waiting on hotosm/basemap-api project to replace this
+
     Args:
         db (Connection): The database connection.
         project_id (int): ID of project to create tiles for.
+        org_id (int): Organisation ID that the project falls within.
         background_task_id (uuid.UUID): UUID of background task to track.
         source (str): Tile source ("esri", "bing", "google", "custom" (tms)).
         output_format (str, optional): Default "mbtiles".
@@ -711,8 +715,6 @@ def generate_project_basemap(
     # NOTE mbtile max supported zoom level is 22 (in GDAL at least)
     zooms = "12-22" if tms else "12-19"
     tiles_dir = f"{TILESDIR}/{project_id}"
-    # FIXME for now this is still a location on disk and we do not upload to S3
-    # FIXME waiting on basemap-api project to replace this with URL
     outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
 
     # NOTE here we put the connection in autocommit mode to ensure we get
@@ -730,7 +732,6 @@ def generate_project_basemap(
                 background_task_id=background_task_id,
                 status=BackgroundTaskStatus.PENDING,
                 tile_source=source,
-                url=outfile,
             ),
         )
 
@@ -761,11 +762,37 @@ def generate_project_basemap(
         )
         log.info(f"Basemap created for project ID {project_id}: {outfile}")
 
+        # Generate S3 urls
+        # We parse as BasemapOut to calculated computed fields (format, mimetype)
+        basemap_out = project_schemas.BasemapOut(
+            **new_basemap.model_dump(exclude=["url"]),
+            url=outfile,
+        )
+        basemap_s3_path = (
+            f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}"
+        )
+        log.debug(f"Uploading basemap to S3 path: {basemap_s3_path}")
+        add_file_to_bucket(
+            settings.S3_BUCKET_NAME,
+            basemap_s3_path,
+            outfile,
+            content_type=basemap_out.mimetype,
+        )
+        basemap_external_s3_url = (
+            f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}/{basemap_s3_path}"
+        )
+        log.info(f"Upload of basemap to S3 complete: {basemap_external_s3_url}")
+        # Delete file on disk
+        Path(outfile).unlink(missing_ok=True)
+
         update_basemap_sync = async_to_sync(DbBasemap.update)
         update_basemap_sync(
             db,
-            new_basemap.id,
-            project_schemas.BasemapUpdate(status=BackgroundTaskStatus.SUCCESS),
+            basemap_out.id,
+            project_schemas.BasemapUpdate(
+                url=basemap_external_s3_url,
+                status=BackgroundTaskStatus.SUCCESS,
+            ),
         )
 
         update_bg_task_sync = async_to_sync(DbBackgroundTask.update)

src/backend/app/projects/project_routes.py

@@ -22,7 +22,6 @@
 from io import BytesIO
 from pathlib import Path
 from typing import Annotated, Optional
-from uuid import UUID
 
 import requests
 from fastapi import (
@@ -37,7 +36,7 @@
     UploadFile,
 )
 from fastapi.concurrency import run_in_threadpool
-from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
+from fastapi.responses import JSONResponse, StreamingResponse
 from fmtm_splitter.splitter import split_by_sql, split_by_square
 from geojson_pydantic import FeatureCollection
 from loguru import logger as log
@@ -314,42 +313,45 @@ async def tiles_list(
     return await DbBasemap.all(db, project_user.get("project").id)
 
 
-@router.get(
-    "/{project_id}/tiles/{tile_id}",
-    response_model=project_schemas.BasemapOut,
-)
-async def download_tiles(
-    tile_id: UUID,
-    db: Annotated[Connection, Depends(db_conn)],
-    project_user: Annotated[ProjectUserDict, Depends(mapper)],
-):
-    """Download the basemap tile archive for a project."""
-    log.debug("Getting basemap path from DB")
-    try:
-        db_basemap = await DbBasemap.one(db, tile_id)
-    except KeyError as e:
-        raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) from e
-
-    log.info(f"User requested download for tiles: {db_basemap.url}")
-
-    project = project_user.get("project")
-    filename = Path(db_basemap.url).name.replace(f"{project.id}_", f"{project.slug}_")
-    log.debug(f"Sending tile archive to user: {filename}")
-
-    if db_basemap.format == "mbtiles":
-        mimetype = "application/vnd.mapbox-vector-tile"
-    elif db_basemap.format == "pmtiles":
-        mimetype = "application/vnd.pmtiles"
-    else:
-        mimetype = "application/vnd.sqlite3"
-
-    return FileResponse(
-        db_basemap.url,
-        headers={
-            "Content-Disposition": f"attachment; filename={filename}",
-            "Content-Type": mimetype,
-        },
-    )
+# NOTE we no longer need this as tiles are uploaded to S3
+# However, it could be useful if requiring private buckets in
+# the future, with pre-signed URL generation
+# @router.get(
+#     "/{project_id}/tiles/{tile_id}",
+#     response_model=project_schemas.BasemapOut,
+# )
+# async def download_tiles(
+#     tile_id: UUID,
+#     db: Annotated[Connection, Depends(db_conn)],
+#     project_user: Annotated[ProjectUserDict, Depends(mapper)],
+# ):
+#     """Download the basemap tile archive for a project."""
+#     log.debug("Getting basemap path from DB")
+#     try:
+#         db_basemap = await DbBasemap.one(db, tile_id)
+#     except KeyError as e:
+#         raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) from e
+
+#     log.info(f"User requested download for tiles: {db_basemap.url}")
+
+#     project = project_user.get("project")
+#     filename = Path(db_basemap.url).name.replace(f"{project.id}_", f"{project.slug}_")
+#     log.debug(f"Sending tile archive to user: {filename}")
+
+#     if db_basemap.format == "mbtiles":
+#         mimetype = "application/vnd.mapbox-vector-tile"
+#     elif db_basemap.format == "pmtiles":
+#         mimetype = "application/vnd.pmtiles"
+#     else:
+#         mimetype = "application/vnd.sqlite3"
+
+#     return FileResponse(
+#         db_basemap.url,
+#         headers={
+#             "Content-Disposition": f"attachment; filename={filename}",
+#             "Content-Type": mimetype,
+#         },
+#     )
 
 
 @router.get("/categories")
@@ -956,6 +958,7 @@ async def generate_project_basemap(
 ):
     """Returns basemap tiles for a project."""
     project_id = project_user.get("project").id
+    org_id = project_user.get("project").organisation_id
 
     # Create task in db and return uuid
     log.debug(
@@ -974,6 +977,7 @@ async def generate_project_basemap(
         project_crud.generate_project_basemap,
         db,
         project_id,
+        org_id,
         background_task_id,
         basemap_in.tile_source,
         basemap_in.file_format,

src/backend/app/projects/project_schemas.py

@@ -286,9 +286,9 @@ class BasemapIn(DbBasemap):
     id: Annotated[Optional[UUID], Field(exclude=True)] = None
     project_id: int
     tile_source: str
-    url: str
     background_task_id: UUID
     status: BackgroundTaskStatus
+    # 'url' not set to mandatory, as it can be updated after upload
 
 
 class BasemapUpdate(DbBasemap):

src/backend/app/s3.py

@@ -67,20 +67,26 @@ def object_exists(bucket_name: str, s3_path: str) -> bool:
             ) from e
 
 
-def add_file_to_bucket(bucket_name: str, file_path: str, s3_path: str):
+def add_file_to_bucket(
+    bucket_name: str,
+    s3_path: str,
+    file_path: str,
+    content_type="application/octet-stream",
+):
     """Upload a file from the filesystem to an S3 bucket.
 
     Args:
         bucket_name (str): The name of the S3 bucket.
-        file_path (str): The path to the file on the local filesystem.
         s3_path (str): The path in the S3 bucket where the file will be stored.
+        file_path (str): The path to the file on the local filesystem.
+        content_type (str): The file mimetype, default application/octet-stream.
     """
     # Ensure s3_path starts with a forward slash
     if not s3_path.startswith("/"):
         s3_path = f"/{s3_path}"
 
     client = s3_client()
-    client.fput_object(bucket_name, file_path, s3_path)
+    client.fput_object(bucket_name, s3_path, file_path, content_type=content_type)
 
 
 def add_obj_to_bucket(
@@ -110,7 +116,12 @@ def add_obj_to_bucket(
     file_obj.seek(0)
 
     result = client.put_object(
-        bucket_name, s3_path, file_obj, file_obj.getbuffer().nbytes, **kwargs
+        bucket_name,
+        s3_path,
+        file_obj,
+        file_obj.getbuffer().nbytes,
+        content_type=content_type,
+        **kwargs,
     )
     log.debug(
         f"Created {result.object_name} object; etag: {result.etag}, "