diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index 6f074c116e..72b1055533 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -1523,6 +1523,23 @@ async def update( return updated_task + @classmethod + async def delete(cls, db: Connection, background_task_id: UUID) -> bool: + """Delete a background task entry.""" + sql = """ + DELETE from background_tasks + WHERE id = %(background_task_id)s + RETURNING id; + """ + + async with db.cursor() as cur: + await cur.execute(sql, {"background_task_id": background_task_id}) + success = await cur.fetchone() + + if success: + return True + return False + class DbBasemap(BaseModel): """Table tiles_path. @@ -1636,7 +1653,7 @@ async def create( async def update( cls, db: Connection, - basemap_id: int, + basemap_id: UUID, basemap_update: "BasemapUpdate", ) -> Self: """Update values for a basemap.""" @@ -1662,6 +1679,23 @@ async def update( return updated_basemap + @classmethod + async def delete(cls, db: Connection, basemap_id: UUID) -> bool: + """Delete a basemap.""" + sql = """ + DELETE from basemaps + WHERE id = %(basemap_id)s + RETURNING id; + """ + + async with db.cursor() as cur: + await cur.execute(sql, {"basemap_id": basemap_id}) + success = await cur.fetchone() + + if success: + return True + return False + class DbSubmissionPhoto(BaseModel): """Table submission_photo. diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 98bbe6fffd..e3cfb29532 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -49,7 +49,7 @@ split_geojson_by_task_areas, ) from app.projects import project_deps, project_schemas -from app.s3 import add_obj_to_bucket +from app.s3 import add_file_to_bucket, add_obj_to_bucket TILESDIR = "/opt/tiles" @@ -686,6 +686,7 @@ async def get_project_features_geojson( def generate_project_basemap( db: Connection, project_id: int, + org_id: int, background_task_id: uuid.UUID, source: str, output_format: str = "mbtiles", @@ -693,9 +694,12 @@ def generate_project_basemap( ): """Get the tiles for a project. + FIXME waiting on hotosm/basemap-api project to replace this + Args: db (Connection): The database connection. project_id (int): ID of project to create tiles for. + org_id (int): Organisation ID that the project falls within. background_task_id (uuid.UUID): UUID of background task to track. source (str): Tile source ("esri", "bing", "google", "custom" (tms)). output_format (str, optional): Default "mbtiles". @@ -711,8 +715,6 @@ def generate_project_basemap( # NOTE mbtile max supported zoom level is 22 (in GDAL at least) zooms = "12-22" if tms else "12-19" tiles_dir = f"{TILESDIR}/{project_id}" - # FIXME for now this is still a location on disk and we do not upload to S3 - # FIXME waiting on basemap-api project to replace this with URL outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}" # NOTE here we put the connection in autocommit mode to ensure we get @@ -730,7 +732,6 @@ def generate_project_basemap( background_task_id=background_task_id, status=BackgroundTaskStatus.PENDING, tile_source=source, - url=outfile, ), ) @@ -761,11 +762,37 @@ def generate_project_basemap( ) log.info(f"Basemap created for project ID {project_id}: {outfile}") + # Generate S3 urls + # We parse as BasemapOut to calculated computed fields (format, mimetype) + basemap_out = project_schemas.BasemapOut( + **new_basemap.model_dump(exclude=["url"]), + url=outfile, + ) + basemap_s3_path = ( + f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}" + ) + log.debug(f"Uploading basemap to S3 path: {basemap_s3_path}") + add_file_to_bucket( + settings.S3_BUCKET_NAME, + basemap_s3_path, + outfile, + content_type=basemap_out.mimetype, + ) + basemap_external_s3_url = ( + f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}/{basemap_s3_path}" + ) + log.info(f"Upload of basemap to S3 complete: {basemap_external_s3_url}") + # Delete file on disk + Path(outfile).unlink(missing_ok=True) + update_basemap_sync = async_to_sync(DbBasemap.update) update_basemap_sync( db, - new_basemap.id, - project_schemas.BasemapUpdate(status=BackgroundTaskStatus.SUCCESS), + basemap_out.id, + project_schemas.BasemapUpdate( + url=basemap_external_s3_url, + status=BackgroundTaskStatus.SUCCESS, + ), ) update_bg_task_sync = async_to_sync(DbBackgroundTask.update) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 67eae45d70..d4b7ef7243 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -22,7 +22,6 @@ from io import BytesIO from pathlib import Path from typing import Annotated, Optional -from uuid import UUID import requests from fastapi import ( @@ -37,7 +36,7 @@ UploadFile, ) from fastapi.concurrency import run_in_threadpool -from fastapi.responses import FileResponse, JSONResponse, StreamingResponse +from fastapi.responses import JSONResponse, StreamingResponse from fmtm_splitter.splitter import split_by_sql, split_by_square from geojson_pydantic import FeatureCollection from loguru import logger as log @@ -314,42 +313,45 @@ async def tiles_list( return await DbBasemap.all(db, project_user.get("project").id) -@router.get( - "/{project_id}/tiles/{tile_id}", - response_model=project_schemas.BasemapOut, -) -async def download_tiles( - tile_id: UUID, - db: Annotated[Connection, Depends(db_conn)], - project_user: Annotated[ProjectUserDict, Depends(mapper)], -): - """Download the basemap tile archive for a project.""" - log.debug("Getting basemap path from DB") - try: - db_basemap = await DbBasemap.one(db, tile_id) - except KeyError as e: - raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) from e - - log.info(f"User requested download for tiles: {db_basemap.url}") - - project = project_user.get("project") - filename = Path(db_basemap.url).name.replace(f"{project.id}_", f"{project.slug}_") - log.debug(f"Sending tile archive to user: {filename}") - - if db_basemap.format == "mbtiles": - mimetype = "application/vnd.mapbox-vector-tile" - elif db_basemap.format == "pmtiles": - mimetype = "application/vnd.pmtiles" - else: - mimetype = "application/vnd.sqlite3" - - return FileResponse( - db_basemap.url, - headers={ - "Content-Disposition": f"attachment; filename={filename}", - "Content-Type": mimetype, - }, - ) +# NOTE we no longer need this as tiles are uploaded to S3 +# However, it could be useful if requiring private buckets in +# the future, with pre-signed URL generation +# @router.get( +# "/{project_id}/tiles/{tile_id}", +# response_model=project_schemas.BasemapOut, +# ) +# async def download_tiles( +# tile_id: UUID, +# db: Annotated[Connection, Depends(db_conn)], +# project_user: Annotated[ProjectUserDict, Depends(mapper)], +# ): +# """Download the basemap tile archive for a project.""" +# log.debug("Getting basemap path from DB") +# try: +# db_basemap = await DbBasemap.one(db, tile_id) +# except KeyError as e: +# raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) from e + +# log.info(f"User requested download for tiles: {db_basemap.url}") + +# project = project_user.get("project") +# filename = Path(db_basemap.url).name.replace(f"{project.id}_", f"{project.slug}_") +# log.debug(f"Sending tile archive to user: {filename}") + +# if db_basemap.format == "mbtiles": +# mimetype = "application/vnd.mapbox-vector-tile" +# elif db_basemap.format == "pmtiles": +# mimetype = "application/vnd.pmtiles" +# else: +# mimetype = "application/vnd.sqlite3" + +# return FileResponse( +# db_basemap.url, +# headers={ +# "Content-Disposition": f"attachment; filename={filename}", +# "Content-Type": mimetype, +# }, +# ) @router.get("/categories") @@ -956,6 +958,7 @@ async def generate_project_basemap( ): """Returns basemap tiles for a project.""" project_id = project_user.get("project").id + org_id = project_user.get("project").organisation_id # Create task in db and return uuid log.debug( @@ -974,6 +977,7 @@ async def generate_project_basemap( project_crud.generate_project_basemap, db, project_id, + org_id, background_task_id, basemap_in.tile_source, basemap_in.file_format, diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 9a45279226..7e7aa8676a 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -286,9 +286,9 @@ class BasemapIn(DbBasemap): id: Annotated[Optional[UUID], Field(exclude=True)] = None project_id: int tile_source: str - url: str background_task_id: UUID status: BackgroundTaskStatus + # 'url' not set to mandatory, as it can be updated after upload class BasemapUpdate(DbBasemap): diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index a280f385f1..9c71a232ef 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -67,20 +67,26 @@ def object_exists(bucket_name: str, s3_path: str) -> bool: ) from e -def add_file_to_bucket(bucket_name: str, file_path: str, s3_path: str): +def add_file_to_bucket( + bucket_name: str, + s3_path: str, + file_path: str, + content_type="application/octet-stream", +): """Upload a file from the filesystem to an S3 bucket. Args: bucket_name (str): The name of the S3 bucket. - file_path (str): The path to the file on the local filesystem. s3_path (str): The path in the S3 bucket where the file will be stored. + file_path (str): The path to the file on the local filesystem. + content_type (str): The file mimetype, default application/octet-stream. """ # Ensure s3_path starts with a forward slash if not s3_path.startswith("/"): s3_path = f"/{s3_path}" client = s3_client() - client.fput_object(bucket_name, file_path, s3_path) + client.fput_object(bucket_name, s3_path, file_path, content_type=content_type) def add_obj_to_bucket( @@ -110,7 +116,12 @@ def add_obj_to_bucket( file_obj.seek(0) result = client.put_object( - bucket_name, s3_path, file_obj, file_obj.getbuffer().nbytes, **kwargs + bucket_name, + s3_path, + file_obj, + file_obj.getbuffer().nbytes, + content_type=content_type, + **kwargs, ) log.debug( f"Created {result.object_name} object; etag: {result.etag}, " diff --git a/src/backend/migrations/qrcode_to_odktoken.py b/src/backend/migrations/archived/2024-08-09/qrcode_to_odktoken.py similarity index 100% rename from src/backend/migrations/qrcode_to_odktoken.py rename to src/backend/migrations/archived/2024-08-09/qrcode_to_odktoken.py diff --git a/src/backend/migrations/upload_basemaps_s3.py b/src/backend/migrations/upload_basemaps_s3.py new file mode 100644 index 0000000000..c3d7e1c00c --- /dev/null +++ b/src/backend/migrations/upload_basemaps_s3.py @@ -0,0 +1,134 @@ +"""Upload basemaps on the filesystem to the connected S3 bucket.""" + +import asyncio +from pathlib import Path + +from dotenv import load_dotenv +from psycopg import AsyncConnection + +from app.config import settings +from app.db.models import DbBackgroundTask, DbBasemap, DbProject +from app.projects.project_schemas import BasemapOut, BasemapUpdate +from app.s3 import add_file_to_bucket + +# Load environment variables +load_dotenv(Path(__file__).parent.parent / ".env") + + +async def upload_basemap_file( + project_id: int, org_id: int, basemap: DbBasemap, filepath: str +) -> str: + """Upload the file to S3 and cleanup the file after.""" + try: + basemap_out = BasemapOut( + **basemap.model_dump(exclude={"url"}), + url=filepath, + ) + basemap_s3_path = ( + f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}" + ) + print(f"Uploading basemap to S3 path: {basemap_s3_path}") + add_file_to_bucket( + settings.S3_BUCKET_NAME, + basemap_s3_path, + filepath, + content_type=basemap_out.mimetype, + ) + basemap_external_s3_url = ( + f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}/{basemap_s3_path}" + ) + print(f"Upload of basemap to S3 complete: {basemap_external_s3_url}") + # Delete file on disk + Path(filepath).unlink(missing_ok=True) + + return basemap_external_s3_url + except Exception as e: + print(f"Error uploading basemap file: {e}") + raise + + +async def basemap_files_to_s3(): + """Upload basemap file & update the DbBasemap.url field.""" + async with await AsyncConnection.connect( + settings.FMTM_DB_URL.unicode_string(), + ) as db: + try: + projects = await DbProject.all(db) + if not projects: + print("No projects found.") + return + + for project in projects: + print("") + print("--------------------------") + print(f"------- Project {project.id} -------") + print("--------------------------") + print("") + basemaps = await DbBasemap.all(db, project.id) + if not basemaps: + print("No basemaps found for upload.") + continue + + for index, basemap in enumerate(basemaps): + status = basemap.status + filepath = basemap.url + + if status == "FAILED": + print( + f"({index}) Cleanup / remove FAILED " + f"basemap creation ({basemap.id})" + ) + await DbBasemap.delete(db, basemap.id) + if not basemap.background_task_id: + continue + + print( + f"Also removing related DbBackgroundTask entry " + f"({basemap.background_task_id})" + ) + await DbBackgroundTask.delete(db, basemap.background_task_id) + continue + + if filepath and filepath.startswith("http"): + print( + f"({index}) Basemap already uploaded " + f"({basemap.id}): {basemap.url}" + ) + continue + + elif not filepath or not Path(filepath).exists(): + print( + f"({index}) File does not exist or invalid " + f"filepath for basemap ({basemap.id}): {basemap.url}" + ) + continue + + try: + new_s3_path = await upload_basemap_file( + project.id, + project.organisation_id, + basemap, + filepath, + ) + await DbBasemap.update( + db, + basemap.id, + BasemapUpdate(url=new_s3_path), + ) + print( + f"({index}) Basemap {basemap.id} " + f"URL updated to {new_s3_path}" + ) + except Exception as e: + print( + f"Failed to process basemap {basemap.id} " + f"for project {project.id}: {e}" + ) + except Exception as e: + print(f"Error in basemap_files_to_s3: {e}") + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(basemap_files_to_s3()) + loop.close() diff --git a/src/frontend/package.json b/src/frontend/package.json index 3b88535d87..225da2fa82 100755 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -87,6 +87,7 @@ "ol": "^8.0.0", "ol-ext": "^4.0.11", "ol-layerswitcher": "^4.1.1", + "ol-pmtiles": "^1.0.2", "pako": "^2.1.0", "pmtiles": "^3.0.6", "qrcode-generator": "^1.4.4", diff --git a/src/frontend/pnpm-lock.yaml b/src/frontend/pnpm-lock.yaml index 41c6f6bb13..21ed459a22 100644 --- a/src/frontend/pnpm-lock.yaml +++ b/src/frontend/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: ol-layerswitcher: specifier: ^4.1.1 version: 4.1.1(ol@8.2.0) + ol-pmtiles: + specifier: ^1.0.2 + version: 1.0.2(ol@8.2.0) pako: specifier: ^2.1.0 version: 2.1.0 @@ -3566,6 +3569,11 @@ packages: peerDependencies: ol: '>=5.0.0' + ol-pmtiles@1.0.2: + resolution: {integrity: sha512-+2itEeTcOk6RWikH2/cWIvv2mFW0empLaCibon65e1kyOEzB+zHIqF3eKa15yyznV8r9K0wfx9S3aG6ceXL0hQ==} + peerDependencies: + ol: '>=9.0.0' + ol@8.2.0: resolution: {integrity: sha512-/m1ddd7Jsp4Kbg+l7+ozR5aKHAZNQOBAoNZ5pM9Jvh4Etkf0WGkXr9qXd7PnhmwiC1Hnc2Toz9XjCzBBvexfXw==} @@ -3676,6 +3684,9 @@ packages: pmtiles@3.0.6: resolution: {integrity: sha512-IdeMETd5lBIDVTLul1HFl0Q7l4KLJjzdxgcp+sN7pYvbipaV7o/0u0HiV06kaFCD0IGEN8KtUHyFZpY30WMflw==} + pmtiles@3.2.1: + resolution: {integrity: sha512-3R4fBwwoli5mw7a6t1IGwOtfmcSAODq6Okz0zkXhS1zi9sz1ssjjIfslwPvcWw5TNhdjNBUg9fgfPLeqZlH6ng==} + possible-typed-array-names@1.0.0: resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} engines: {node: '>= 0.4'} @@ -8432,6 +8443,11 @@ snapshots: dependencies: ol: 8.2.0 + ol-pmtiles@1.0.2(ol@8.2.0): + dependencies: + ol: 8.2.0 + pmtiles: 3.2.1 + ol@8.2.0: dependencies: color-rgba: 3.0.0 @@ -8541,6 +8557,11 @@ snapshots: '@types/leaflet': 1.9.8 fflate: 0.8.2 + pmtiles@3.2.1: + dependencies: + '@types/leaflet': 1.9.8 + fflate: 0.8.2 + possible-typed-array-names@1.0.0: {} postcss-import@15.1.0(postcss@8.4.38): diff --git a/src/frontend/src/api/Files.ts b/src/frontend/src/api/Files.ts index f130f68186..e6866728f2 100755 --- a/src/frontend/src/api/Files.ts +++ b/src/frontend/src/api/Files.ts @@ -55,69 +55,3 @@ export const GetProjectQrCode = ( }, [projectName, odkToken, osmUser]); return { qrcode }; }; - -export async function readFileFromOPFS(filePath: string) { - const opfsRoot = await navigator.storage.getDirectory(); - const directories = filePath.split('/'); - - let currentDirectoryHandle = opfsRoot; - - // Iterate dirs and get directoryHandles - for (const directory of directories.slice(0, -1)) { - console.log(`Reading OPFS dir: ${directory}`); - try { - currentDirectoryHandle = await currentDirectoryHandle.getDirectoryHandle(directory); - } catch { - return null; // Directory doesn't exist - } - } - - // Get file within final directory handle - try { - const filename: any = directories.pop(); - console.log(`Getting OPFS file: ${filename}`); - const fileHandle = await currentDirectoryHandle.getFileHandle(filename); - const fileData = await fileHandle.getFile(); // Read the file - return fileData; - } catch { - return null; // File doesn't exist or error occurred - } -} - -export async function writeBinaryToOPFS(filePath: string, data: any) { - console.log(`Starting write to OPFS file: ${filePath}`); - - const opfsRoot = await navigator.storage.getDirectory(); - - // Split the filePath into directories and filename - const directories = filePath.split('/'); - const filename: any = directories.pop(); - - // Start with the root directory handle - let currentDirectoryHandle = opfsRoot; - - // Iterate over directories and create nested directories - for (const directory of directories) { - console.log(`Creating OPFS dir: ${directory}`); - try { - currentDirectoryHandle = await currentDirectoryHandle.getDirectoryHandle(directory, { create: true }); - } catch (error) { - console.error('Error creating directory:', error); - } - } - - // Create the file handle within the last directory - const fileHandle = await currentDirectoryHandle.getFileHandle(filename, { create: true }); - const writable = await fileHandle.createWritable(); - - // Write data to the writable stream - try { - await writable.write(data); - } catch (error) { - console.log(error); - } - - // Close the writable stream - await writable.close(); - console.log(`Finished write to OPFS file: ${filePath}`); -} diff --git a/src/frontend/src/api/Project.ts b/src/frontend/src/api/Project.ts index c994ff8fd2..ac14e5f84f 100755 --- a/src/frontend/src/api/Project.ts +++ b/src/frontend/src/api/Project.ts @@ -2,8 +2,6 @@ import { ProjectActions } from '@/store/slices/ProjectSlice'; import { CommonActions } from '@/store/slices/CommonSlice'; import CoreModules from '@/shared/CoreModules'; import { task_state, task_event } from '@/types/enums'; -import { writeBinaryToOPFS } from '@/api/Files'; -import { projectInfoType } from '@/models/project/projectModel'; export const ProjectById = (projectId: string) => { return async (dispatch) => { @@ -169,51 +167,23 @@ export const GenerateProjectTiles = (url: string, projectId: string, data: objec }; }; -export const DownloadTile = (url: string, payload: Partial, toOpfs: boolean = false) => { +export const DownloadBasemapFile = (url: string) => { return async (dispatch) => { - dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: true })); + dispatch(ProjectActions.SetDownloadTileLoading({ loading: true })); - const getDownloadTile = async (url: string, payload: Partial, toOpfs: boolean) => { + const downloadBasemapFromAPI = async (url: string) => { try { - const response = await CoreModules.axios.get(url, { - responseType: 'arraybuffer', - }); - - // Get filename from Content-Disposition header - const tileData = response.data; - - if (toOpfs) { - // Copy to OPFS filesystem for offline use - const projectId = payload.id; - const filePath = `${projectId}/all.pmtiles`; - await writeBinaryToOPFS(filePath, tileData); - // Set the OPFS file path to project state - dispatch(ProjectActions.SetProjectOpfsBasemapPath(filePath)); - return; - } - - const filename = response.headers['content-disposition'].split('filename=')[1]; - console.log(filename); - // Create Blob from ArrayBuffer - const blob = new Blob([tileData], { type: response.headers['content-type'] }); - const downloadUrl = URL.createObjectURL(blob); - - const a = document.createElement('a'); - a.href = downloadUrl; - a.download = filename; - a.click(); - - // Clean up object URL - URL.revokeObjectURL(downloadUrl); + // Open S3 url directly + window.open(url); - dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false })); + dispatch(ProjectActions.SetDownloadTileLoading({ loading: false })); } catch (error) { - dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false })); + dispatch(ProjectActions.SetDownloadTileLoading({ loading: false })); } finally { - dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false })); + dispatch(ProjectActions.SetDownloadTileLoading({ loading: false })); } }; - await getDownloadTile(url, payload, toOpfs); + await downloadBasemapFromAPI(url); }; }; diff --git a/src/frontend/src/components/GenerateBasemap.tsx b/src/frontend/src/components/GenerateBasemap.tsx index dde23eeedc..a547974031 100644 --- a/src/frontend/src/components/GenerateBasemap.tsx +++ b/src/frontend/src/components/GenerateBasemap.tsx @@ -3,7 +3,7 @@ import CoreModules from '@/shared/CoreModules'; import AssetModules from '@/shared/AssetModules'; import { CommonActions } from '@/store/slices/CommonSlice'; import environment from '@/environment'; -import { DownloadTile, GenerateProjectTiles, GetTilesList } from '@/api/Project'; +import { DownloadBasemapFile, GenerateProjectTiles, GetTilesList, ProjectById } from '@/api/Project'; import { ProjectActions } from '@/store/slices/ProjectSlice'; import { projectInfoType } from '@/models/project/projectModel'; import { useAppSelector } from '@/types/reduxTypes'; @@ -31,9 +31,6 @@ const GenerateBasemap = ({ projectInfo }: { projectInfo: Partial { - dispatch(DownloadTile(`${import.meta.env.VITE_API_URL}/projects/${id}/tiles/${tileId}`, projectInfo, toOpfs)); - }; const getTilesList = () => { dispatch(GetTilesList(`${import.meta.env.VITE_API_URL}/projects/${id}/tiles`)); @@ -306,16 +303,16 @@ const GenerateBasemap = ({ projectInfo }: { projectInfo: Partial
{list.status === 'SUCCESS' && list.format === 'pmtiles' && ( - downloadBasemap(list.id, true)} + onClick={() => dispatch(ProjectActions.SetPmtileBasemapUrl(list.url))} className="fmtm-text-red-500 hover:fmtm-text-red-700" /> )} {list.status === 'SUCCESS' && ( downloadBasemap(list.id)} + onClick={() => dispatch(DownloadBasemapFile(list.url))} className="fmtm-text-gray-500 hover:fmtm-text-blue-500" /> )} diff --git a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/LayerSwitchMenu.tsx b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/LayerSwitchMenu.tsx index 3f13dc8427..55fc00f365 100644 --- a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/LayerSwitchMenu.tsx +++ b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/LayerSwitchMenu.tsx @@ -56,7 +56,7 @@ const LayerCard = ({ layer, changeBaseLayerHandler, active }: layerCardPropType) ); }; -const LayerSwitchMenu = ({ map, pmTileLayerData = null }: { map: any; pmTileLayerData?: any }) => { +const LayerSwitchMenu = ({ map, pmTileLayerUrl = null }: { map: any; pmTileLayerUrl?: any }) => { const { pathname } = useLocation(); const [baseLayers, setBaseLayers] = useState(['OSM', 'Satellite', 'None']); const [hasPMTile, setHasPMTile] = useState(false); @@ -78,10 +78,10 @@ const LayerSwitchMenu = ({ map, pmTileLayerData = null }: { map: any; pmTileLaye }, [projectInfo, pathname, map]); useEffect(() => { - if (!pmTileLayerData || baseLayers.includes('PMTile')) return; + if (!pmTileLayerUrl || baseLayers.includes('Custom')) return; setHasPMTile(true); - setActiveTileLayer('PMTile'); - }, [pmTileLayerData]); + setActiveTileLayer('Custom'); + }, [pmTileLayerUrl]); const changeBaseLayer = (baseLayerTitle: string) => { const allLayers = map.getLayers(); @@ -90,7 +90,7 @@ const LayerSwitchMenu = ({ map, pmTileLayerData = null }: { map: any; pmTileLaye ); const baseLayersCollection: Record[] = filteredBaseLayers?.values_?.layers.array_; baseLayersCollection - ?.filter((bLayer) => bLayer?.values_?.title !== 'PMTile') + ?.filter((bLayer) => bLayer?.values_?.title !== 'Custom') ?.forEach((baseLayer) => { if (baseLayer?.values_?.title === baseLayerTitle) { baseLayer.setVisible(true); @@ -158,10 +158,10 @@ const LayerSwitchMenu = ({ map, pmTileLayerData = null }: { map: any; pmTileLaye
Tiles
toggleTileLayer('PMTile')} - active={'PMTile' === activeTileLayer} + key="Custom" + layer="Custom" + changeBaseLayerHandler={() => toggleTileLayer('Custom')} + active={'Custom' === activeTileLayer} />
diff --git a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js index 0d3a823e2c..7bf5798c2b 100644 --- a/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js +++ b/src/frontend/src/components/MapComponent/OpenLayersComponent/LayerSwitcher/index.js @@ -8,9 +8,8 @@ import LayerSwitcher from 'ol-layerswitcher'; import React, { useEffect, useState } from 'react'; import { XYZ } from 'ol/source'; import { useLocation } from 'react-router-dom'; -import DataTile from 'ol/source/DataTile.js'; -import TileLayer from 'ol/layer/WebGLTile.js'; -import { FileSource, PMTiles } from 'pmtiles'; +import WebGLTile from 'ol/layer/WebGLTile.js'; +import { PMTilesRasterSource } from 'ol-pmtiles'; import windowDimention from '@/hooks/WindowDimension'; import { useAppSelector } from '@/types/reduxTypes'; @@ -134,41 +133,20 @@ const monochromeMidNight = (visible = false) => }), }); -const pmTileLayer = (pmTileLayerData, visible) => { - function loadImage(src) { - return new Promise((resolve, reject) => { - const img = new Image(); - img.addEventListener('load', () => resolve(img)); - img.addEventListener('error', () => reject(new Error('load failed'))); - img.src = src; - }); - } - - const pmTiles = new PMTiles(new FileSource(pmTileLayerData)); - - async function loader(z, x, y) { - const response = await pmTiles.getZxy(z, x, y); - const blob = new Blob([response.data]); - const src = URL.createObjectURL(blob); - const image = await loadImage(src); - URL.revokeObjectURL(src); - return image; - } - return new TileLayer({ - title: `PMTile`, - type: 'raster pm tiles', - visible: true, - source: new DataTile({ - loader, - wrapX: true, +const pmTileLayer = (pmTileLayerUrl, visible = true) => { + return new WebGLTile({ + title: 'Custom', + type: 'custom', + visible: visible, + source: new PMTilesRasterSource({ + url: pmTileLayerUrl, + attributions: ['OpenAerialMap'], tileSize: [512, 512], - maxZoom: 22, - attributions: 'Tiles © OpenAerialMap', }), }); }; -const LayerSwitcherControl = ({ map, visible = 'osm', pmTileLayerData = null }) => { +const LayerSwitcherControl = ({ map, visible = 'osm', pmTileLayerUrl = null }) => { const { windowSize } = windowDimention(); const { pathname } = useLocation(); @@ -181,7 +159,7 @@ const LayerSwitcherControl = ({ map, visible = 'osm', pmTileLayerData = null }) // mapboxMap(visible), // mapboxOutdoors(visible), none(visible), - // pmTileLayer(pmTileLayerData, visible), + // pmTileLayer(pmTileLayerUrl, visible), ], }), ); @@ -246,11 +224,11 @@ const LayerSwitcherControl = ({ map, visible = 'osm', pmTileLayerData = null }) }, [map, visible]); useEffect(() => { - if (!pmTileLayerData) { + if (!pmTileLayerUrl) { return; } - const pmTileBaseLayer = pmTileLayer(pmTileLayerData, visible); + const pmTileBaseLayer = pmTileLayer(pmTileLayerUrl, visible); const currentLayers = basemapLayers.getLayers(); currentLayers.push(pmTileBaseLayer); @@ -259,7 +237,7 @@ const LayerSwitcherControl = ({ map, visible = 'osm', pmTileLayerData = null }) return () => { basemapLayers.getLayers().remove(pmTileBaseLayer); }; - }, [pmTileLayerData]); + }, [pmTileLayerUrl]); const location = useLocation(); useEffect(() => {}, [map]); diff --git a/src/frontend/src/components/ProjectDetailsV2/MapControlComponent.tsx b/src/frontend/src/components/ProjectDetailsV2/MapControlComponent.tsx index 4461329eec..439d2717ec 100644 --- a/src/frontend/src/components/ProjectDetailsV2/MapControlComponent.tsx +++ b/src/frontend/src/components/ProjectDetailsV2/MapControlComponent.tsx @@ -15,7 +15,7 @@ import MapLegends from '@/components/MapLegends'; type mapControlComponentType = { map: any; projectName: string; - pmTileLayerData: any; + pmTileLayerUrl: any; }; const btnList = [ @@ -41,7 +41,7 @@ const btnList = [ }, ]; -const MapControlComponent = ({ map, projectName, pmTileLayerData }: mapControlComponentType) => { +const MapControlComponent = ({ map, projectName, pmTileLayerUrl }: mapControlComponentType) => { const { pathname } = useLocation(); const dispatch = CoreModules.useAppDispatch(); const [toggleCurrentLoc, setToggleCurrentLoc] = useState(false); @@ -89,7 +89,7 @@ const MapControlComponent = ({ map, projectName, pmTileLayerData }: mapControlCo ))} - + {/* download options */}
{ const [dataExtractUrl, setDataExtractUrl] = useState(); const [dataExtractExtent, setDataExtractExtent] = useState(null); const [taskBoundariesLayer, setTaskBoundariesLayer] = useState>(null); - // Can pass a File object, or a string URL to be read by PMTiles - const [customBasemapData, setCustomBasemapData] = useState(); + const customBasemapUrl = useAppSelector((state) => state.project.customBasemapUrl); const [viewState, setViewState] = useState('project_info'); const projectId: string = params.id; const defaultTheme = useAppSelector((state) => state.theme.hotTheme); @@ -68,7 +66,6 @@ const ProjectDetailsV2 = () => { const projectDetailsLoading = useAppSelector((state) => state?.project?.projectDetailsLoading); const geolocationStatus = useAppSelector((state) => state.project.geolocationStatus); const taskModalStatus = CoreModules.useAppSelector((state) => state.project.taskModalStatus); - const projectOpfsBasemapPath = useAppSelector((state) => state?.project?.projectOpfsBasemapPath); const authDetails = CoreModules.useAppSelector((state) => state.login.authDetails); const entityOsmMap = useAppSelector((state) => state?.project?.entityOsmMap); @@ -255,32 +252,6 @@ const ProjectDetailsV2 = () => { dispatch(GetEntityInfo(`${import.meta.env.VITE_API_URL}/projects/${projectId}/entities/statuses`)); }, []); - const getPmtilesBasemap = async () => { - if (!projectOpfsBasemapPath) { - return; - } - const opfsPmtilesData = await readFileFromOPFS(projectOpfsBasemapPath); - setCustomBasemapData(opfsPmtilesData); - }; - useEffect(() => { - if (!projectOpfsBasemapPath) { - return; - } - - // Extract project id from projectOpfsBasemapPath - const projectOpfsBasemapPathParts = projectOpfsBasemapPath.split('/'); - const projectOpfsBasemapProjectId = projectOpfsBasemapPathParts[0]; - - // Check if project id from projectOpfsBasemapPath matches current projectId - if (projectOpfsBasemapProjectId !== projectId) { - // If they don't match, set CustomBasemapData to null - setCustomBasemapData(null); - } else { - // If they match, fetch the basemap data - getPmtilesBasemap(); - } - return () => {}; - }, [projectOpfsBasemapPath]); const [showDebugConsole, setShowDebugConsole] = useState(false); return ( @@ -450,10 +421,7 @@ const ProjectDetailsV2 = () => { />
)} - + {taskBoundariesLayer && taskBoundariesLayer?.features?.length > 0 && ( {
{ - if (extraStyles.length > 0) { + if (styles.length > 0) { + // We do not await this to avoid complicating reactive logic fetchStyleInfo(); } else { allStyles = []; } }); - type MapLibreStylePlusMetadata = maplibregl.StyleSpecification & { - metadata: { - thumbnail?: string; - }; - }; + $effect(() => { + // Set initial selected style + reactiveStyleSelection = allStyles.find((style) => style.name === selectedStyleName) || allStyles[0]; + selectedStyleUrl = reactiveStyleSelection?.metadata?.thumbnail; + }); + + // Update the map when a new style is selected + $effect(() => { + if (reactiveStyleSelection) { + selectStyle(reactiveStyleSelection); + } + }); /** * Extract the raster thumbnail root tile, or return an empty string. @@ -98,72 +115,67 @@ map = new Map({ * Fetch styles and prepare them with thumbnails. */ async function fetchStyleInfo() { + let processedStyles: MapLibreStylePlusMetadata[] = []; + + // Process the current map style const currentMapStyle = map?.getStyle(); if (currentMapStyle) { - const processedStyle = processStyle(currentMapStyle); - selectedStyleUrl = processedStyle?.metadata?.thumbnail || undefined; - allStyles = [processedStyle]; + processedStyles.push(processStyle(currentMapStyle)); } - const extraProcessedStyles = await Promise.all( - extraStyles.map(async (style) => { - if (typeof style === 'string') { - const styleResponse = await fetch(style); - const styleJson = await styleResponse.json(); - return processStyle(styleJson); - } else { - return processStyle(style); - } - }), - ); + // Process additional styles (download first if style is URL) + for (const style of styles) { + if (typeof style === 'string') { + const response = await fetch(style); + const styleJson = await response.json(); + processedStyles.push(processStyle(styleJson)); + } else { + processedStyles.push(processStyle(style)); + } + } - allStyles = allStyles.concat(extraProcessedStyles); + // Deduplicate styles by `name` + allStyles = processedStyles.filter( + (style, index, self) => self.findIndex((s) => s.name === style.name) === index, + ); } function selectStyle(style: MapLibreStylePlusMetadata) { - // returns all the map style i.e. all layers, sources const currentMapStyle = map?.getStyle(); - // reAddLayers: user defined layers that needs to be preserved - const reAddLayers = currentMapStyle?.layers?.filter((layer) => { - return sourcesIdToReAdd?.includes(layer?.source); - }); - - // reAddSources: user defined sources that needs to be preserved - const reAddSources = Object?.entries(currentMapStyle?.sources) - ?.filter(([key]) => sourcesIdToReAdd?.includes(key)) - ?.map(([key, value]) => ({ [key]: value })); + if (style.name === currentMapStyle.name) return; selectedStyleUrl = style.metadata.thumbnail; - // changes to selected base layer (note: user defined layer and sources are lost) + // Apply the selected style to the map + // being sure to save sources and layers to add back on top + const reAddLayers = currentMapStyle?.layers?.filter((layer) => sourcesIdToReAdd.includes(layer.source)); + const reAddSources = Object.entries(currentMapStyle?.sources || {}) + .filter(([key]) => sourcesIdToReAdd.includes(key)) + .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}); + map?.setStyle(style); - isClosed = !isClosed; - - // reapply user defined source - if (reAddSources?.length > 0) { - for (const reAddSource of reAddSources) { - for (const [id, source] of Object.entries(reAddSource)) { - if (!map?.getStyle().sources[id]) { - map?.addSource(id, source); - } + + // Re-add sources and layers + if (reAddSources) { + Object.entries(reAddSources).forEach(([id, source]) => { + if (!map?.getStyle().sources[id]) { + map?.addSource(id, source); } - } + }); } - // reapply user defined layers - if (reAddLayers?.length > 0) { - for (const layer of reAddLayers) { - if (!map?.getStyle().layers.find((layerx) => layerx?.id === layer.id)) { + + if (reAddLayers) { + reAddLayers.forEach((layer) => { + if (!map?.getStyle().layers.find((l) => l.id === layer.id)) { map?.addLayer(layer); } - } + }); } } - onDestroy(() => { allStyles = []; selectedStyleUrl = undefined; - isClosed = true; }); @@ -202,7 +214,8 @@ map = new Map({ > Style Thumbnail {style.name} -
{/each} + + {/each} diff --git a/src/mapper/src/lib/components/map/main.svelte b/src/mapper/src/lib/components/map/main.svelte index 4d67e8c340..59aa05b641 100644 --- a/src/mapper/src/lib/components/map/main.svelte +++ b/src/mapper/src/lib/components/map/main.svelte @@ -2,6 +2,7 @@ import '$styles/page.css'; import '$styles/button.css'; import '@hotosm/ui/dist/hotosm-ui'; + import { onMount, tick } from 'svelte'; import { MapLibre, GeoJSON, @@ -34,8 +35,10 @@ import { getTaskStore } from '$store/tasks.svelte.ts'; import { getProjectSetupStepStore, getProjectBasemapStore } from '$store/common.svelte.ts'; // import { entityFeatcolStore, selectedEntityId } from '$store/entities'; + import { readFileFromOPFS } from '$lib/fs/opfs.ts'; + import { loadOfflinePmtiles } from '$lib/utils/basemaps.ts'; import { projectSetupStep as projectSetupStepEnum } from '$constants/enums.ts'; - import { baseLayers, osmStyle, customStyle } from '$constants/baseLayers.ts'; + import { baseLayers, osmStyle, pmtilesStyle } from '$constants/baseLayers.ts'; import { getEntitiesStatusStore } from '$store/entities.svelte.ts'; import type { GeoJSON as GeoJSONType } from 'geojson'; @@ -58,27 +61,49 @@ let map: maplibregl.Map | undefined = $state(); let loaded: boolean = $state(false); + let selectedBaselayer: string = $state('OSM'); let taskAreaClicked: boolean = $state(false); let toggleGeolocationStatus: boolean = $state(false); let projectSetupStep = $state(null); - // If no custom layer URL, omit, else set URL from projectPmtilesUrl - let processedBaseLayers = $derived([ - ...baseLayers, - ...(projectBasemapStore.projectPmtilesUrl - ? [ - { - ...customStyle, - sources: { - ...customStyle.sources, - custom: { - ...customStyle.sources.custom, - url: projectBasemapStore.projectPmtilesUrl, - }, + // Trigger adding the PMTiles layer to baselayers, if PmtilesUrl is set + let allBaseLayers: maplibregl.StyleSpecification[] = $derived( + projectBasemapStore.projectPmtilesUrl ? + [ + ...baseLayers, + { + ...pmtilesStyle, + sources: { + ...pmtilesStyle.sources, + pmtiles: { + ...pmtilesStyle.sources.pmtiles, + url: projectBasemapStore.projectPmtilesUrl, }, }, - ] - : []), - ]); + }, + ] + : baseLayers + ); + // // This does not work! Infinite looping + // // Trigger adding the PMTiles layer to baselayers, if PmtilesUrl is set + // $effect(() => { + // if (projectBasemapStore.projectPmtilesUrl) { + // const layers = allBaseLayers + // .filter((layer) => layer.name !== "PMTiles") + // .push( + // { + // ...pmtilesStyle, + // sources: { + // ...pmtilesStyle.sources, + // pmtiles: { + // ...pmtilesStyle.sources.pmtiles, + // url: projectBasemapStore.projectPmtilesUrl, + // }, + // }, + // }, + // ) + // allBaseLayers = layers; + // } + // }) // using this function since outside click of entity layer couldn't be tracked via FillLayer function handleMapClick(e: maplibregl.MapMouseEvent) { @@ -137,11 +162,6 @@ $effect(() => { if (map) { setMapRef(map); - // Register pmtiles protocol - if (!maplibre.config.REGISTERED_PROTOCOLS.hasOwnProperty('pmtiles')) { - let protocol = new Protocol(); - maplibre.addProtocol('pmtiles', protocol.tile); - } } }); @@ -173,6 +193,22 @@ }), }; } + + onMount(async () => { + // Register pmtiles protocol + if (!maplibre.config.REGISTERED_PROTOCOLS.hasOwnProperty('pmtiles')) { + let protocol = new Protocol(); + maplibre.addProtocol('pmtiles', protocol.tile); + } + + // Attempt loading OPFS PMTiles layers on first load + // note that this sets projectBasemapStore.projectPmtilesUrl + const offlineBasemapFile = await readFileFromOPFS(`${projectId}/basemap.pmtiles`); + if (offlineBasemapFile) { + await loadOfflinePmtiles(projectId); + selectedBaselayer = 'PMTiles'; + } + }); @@ -213,7 +249,12 @@ - + diff --git a/src/mapper/src/lib/components/offline/basemaps.svelte b/src/mapper/src/lib/components/offline/basemaps.svelte index 37048fe78a..efd381bc08 100644 --- a/src/mapper/src/lib/components/offline/basemaps.svelte +++ b/src/mapper/src/lib/components/offline/basemaps.svelte @@ -1,7 +1,6 @@