feat: selection of basemap output formats, TMS input, mobile UI (#896)

* feat: use create_basemap_file + content-disposition header

* fix: get_project_tiles allow output format arg

* refactor: rename GenerateMbTiles --> GenerateBasemap

* refactor: GenerateMbTiles.jsx --> GenerateBasemap.jsx

* fix(backend): basemap gen use create_basemap_file, allow formats/tms

* fix(frontend): mobile basemap modal, select output format

* fix(frontend): basemap generation tms_url --> tms param

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* refactor: rename downloadMbTiles --> downloadBasemap

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* build: update osm-fieldwork-->0.3.7, osm-rawdata-->0.1.4

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docker-compose.yml

@@ -97,7 +97,7 @@ services:
     entrypoint: ["/migrate-entrypoint.sh"]
     restart: "on-failure:3"
     healthcheck:
-      test: []  # Set the health check test to an empty value to disable it
+      test: [] # Set the health check test to an empty value to disable it
 
   ui:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"

src/backend/app/main.py

@@ -71,6 +71,7 @@ def get_application() -> FastAPI:
         allow_credentials=True,
         allow_methods=["*"],
         allow_headers=["*"],
+        expose_headers=["Content-Disposition"],
     )
 
     _app.include_router(user_routes.router)

src/backend/app/models/enums.py

@@ -222,3 +222,4 @@ class BackgroundTaskStatus(IntEnum, Enum):
 
 
 TILES_SOURCE = ["esri", "bing", "google", "topo"]
+TILES_FORMATS = ["mbtiles", "sqlitedb", "sqlite3", "sqlite", "pmtiles"]

src/backend/app/projects/project_crud.py

@@ -39,7 +39,7 @@
 from geoalchemy2.shape import from_shape
 from geojson import dump
 from loguru import logger as log
-from osm_fieldwork import basemapper
+from osm_fieldwork.basemapper import create_basemap_file
 from osm_fieldwork.data_models import data_models_path
 from osm_fieldwork.filter_data import FilterData
 from osm_fieldwork.json2osm import json2osm
@@ -2300,19 +2300,25 @@ async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str)
 def get_project_tiles(
     db: Session,
     project_id: int,
-    source: str,
     background_task_id: uuid.UUID,
+    source: str,
+    output_format: str = "mbtiles",
+    tms: str = None,
 ):
-    """Get the tiles for a project."""
-    zooms = [12, 13, 14, 15, 16, 17, 18, 19]
-    source = source
+    """Get the tiles for a project.
+
+    Args:
+        project_id (int): ID of project to create tiles for.
+        background_task_id (uuid.UUID): UUID of background task to track.
+        source (str): Tile source ("esri", "bing", "topo", "google", "oam").
+        output_format (str, optional): Default "mbtiles".
+            Other options: "pmtiles", "sqlite3".
+        tms (str, optional): Default None. Custom TMS provider URL.
+    """
+    zooms = "12-19"
     tiles_path_id = uuid.uuid4()
     tiles_dir = f"{TILESDIR}/{tiles_path_id}"
-    base = f"{tiles_dir}/{source}tiles"
-    outfile = f"{tiles_dir}/{project_id}_{source}tiles.mbtiles"
-
-    if not os.path.exists(base):
-        os.makedirs(base)
+    outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
 
     tile_path_instance = db_models.DbTilesPath(
         project_id=project_id,
@@ -2327,45 +2333,45 @@ def get_project_tiles(
         db.commit()
 
         # Project Outline
+        log.debug(f"Getting bbox for project: {project_id}")
         query = text(
-            f"""SELECT jsonb_build_object(
-                    'type', 'FeatureCollection',
-                    'features', jsonb_agg(feature)
-                    )
-                    FROM (
-                    SELECT jsonb_build_object(
-                        'type', 'Feature',
-                        'id', id,
-                        'geometry', ST_AsGeoJSON(outline)::jsonb
-                    ) AS feature
-                    FROM projects
-                    WHERE id={project_id}
-                    ) features;"""
+            f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
+                        ST_YMin(ST_Envelope(outline)) AS min_lat,
+                        ST_XMax(ST_Envelope(outline)) AS max_lon,
+                        ST_YMax(ST_Envelope(outline)) AS max_lat
+                FROM projects
+                WHERE id = {project_id};"""
         )
 
         result = db.execute(query)
-        features = result.fetchone()[0]
-
-        # Boundary
-        boundary_file = f"/tmp/{project_id}_boundary.geojson"
-
-        # Update outfile containing osm extracts with the new geojson contents containing title in the properties.
-        with open(boundary_file, "w") as jsonfile:
-            jsonfile.truncate(0)
-            dump(features, jsonfile)
+        project_bbox = result.fetchone()
+        log.debug(f"Extracted project bbox: {project_bbox}")
 
-        basemap = basemapper.BaseMapper(boundary_file, base, source, False)
-        outf = basemapper.DataFile(outfile, basemap.getFormat())
-        suffix = os.path.splitext(outfile)[1]
-        if suffix == ".mbtiles":
-            outf.addBounds(basemap.bbox)
-        for level in zooms:
-            basemap.getTiles(level)
-            if outfile:
-                # Create output database and specify image format, png, jpg, or tif
-                outf.writeTiles(basemap.tiles, base)
-            else:
-                log.info("Only downloading tiles to %s!" % base)
+        if project_bbox:
+            min_lon, min_lat, max_lon, max_lat = project_bbox
+        else:
+            log.error(f"Failed to get bbox from project: {project_id}")
+
+        log.debug(
+            "Creating basemap with params: "
+            f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
+            f"outfile={outfile} | "
+            f"zooms={zooms} | "
+            f"outdir={tiles_dir} | "
+            f"source={source} | "
+            f"xy={False} | "
+            f"tms={tms}"
+        )
+        create_basemap_file(
+            boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
+            outfile=outfile,
+            zooms=zooms,
+            outdir=tiles_dir,
+            source=source,
+            xy=False,
+            tms=tms,
+        )
+        log.info(f"Basemap created for project ID {project_id}: {outfile}")
 
         tile_path_instance.status = 4
         db.commit()

src/backend/app/projects/project_routes.py

@@ -18,6 +18,7 @@
 import json
 import os
 import uuid
+from pathlib import Path
 from typing import List, Optional
 
 from fastapi import (
@@ -39,7 +40,7 @@
 
 from ..central import central_crud
 from ..db import database, db_models
-from ..models.enums import TILES_SOURCE
+from ..models.enums import TILES_FORMATS, TILES_SOURCE
 from ..tasks import tasks_crud
 from . import project_crud, project_schemas, utils
 from .project_crud import check_crs
@@ -977,16 +978,25 @@ async def generate_project_tiles(
     source: str = Query(
         ..., description="Select a source for tiles", enum=TILES_SOURCE
     ),
+    format: str = Query(
+        "mbtiles", description="Select an output format", enum=TILES_FORMATS
+    ),
+    tms: str = Query(
+        None,
+        description="Provide a custom TMS URL, optional",
+    ),
     db: Session = Depends(database.get_db),
 ):
-    """Returns the tiles for a project.
+    """Returns basemap tiles for a project.
 
     Args:
-        project_id (int): The id of the project.
-        source (str): The selected source.
+        project_id (int): ID of project to create tiles for.
+        source (str): Tile source ("esri", "bing", "topo", "google", "oam").
+        format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3".
+        tms (str, optional): Default None. Custom TMS provider URL.
 
     Returns:
-        Response: The File response object containing the tiles.
+        str: Success message that tile generation started.
     """
     # generate a unique task ID using uuid
     background_task_id = uuid.uuid4()
@@ -997,7 +1007,13 @@ async def generate_project_tiles(
     )
 
     background_tasks.add_task(
-        project_crud.get_project_tiles, db, project_id, source, background_task_id
+        project_crud.get_project_tiles,
+        db,
+        project_id,
+        background_task_id,
+        source,
+        format,
+        tms,
     )
 
     return {"Message": "Tile generation started"}
@@ -1018,14 +1034,24 @@ async def tiles_list(project_id: int, db: Session = Depends(database.get_db)):
 
 @router.get("/download_tiles/")
 async def download_tiles(tile_id: int, db: Session = Depends(database.get_db)):
+    log.debug("Getting tile archive path from DB")
     tiles_path = (
         db.query(db_models.DbTilesPath)
         .filter(db_models.DbTilesPath.id == str(tile_id))
         .first()
     )
+    log.info(f"User requested download for tiles: {tiles_path.path}")
+
+    project_id = tiles_path.project_id
+    project_name = project_crud.get_project(db, project_id).project_name_prefix
+    filename = Path(tiles_path.path).name.replace(
+        f"{project_id}_", f"{project_name.replace(' ', '_')}_"
+    )
+    log.debug(f"Sending tile archive to user: {filename}")
+
     return FileResponse(
         tiles_path.path,
-        headers={"Content-Disposition": "attachment; filename=tiles.mbtiles"},
+        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
     )