From 90ace1b91813cae57c58b874435572ac9f913683 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:49:46 +0100 Subject: [PATCH] feat: selection of basemap output formats, TMS input, mobile UI (#896) * feat: use create_basemap_file + content-disposition header * fix: get_project_tiles allow output format arg * refactor: rename GenerateMbTiles --> GenerateBasemap * refactor: GenerateMbTiles.jsx --> GenerateBasemap.jsx * fix(backend): basemap gen use create_basemap_file, allow formats/tms * fix(frontend): mobile basemap modal, select output format * fix(frontend): basemap generation tms_url --> tms param * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor: rename downloadMbTiles --> downloadBasemap * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * build: update osm-fieldwork-->0.3.7, osm-rawdata-->0.1.4 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docker-compose.yml | 2 +- src/backend/app/main.py | 1 + src/backend/app/models/enums.py | 1 + src/backend/app/projects/project_crud.py | 94 +++---- src/backend/app/projects/project_routes.py | 40 ++- src/backend/pdm.lock | 115 ++++---- src/backend/pyproject.toml | 4 +- src/frontend/src/api/Project.js | 6 +- .../src/components/GenerateBasemap.jsx | 249 ++++++++++++++++++ .../src/components/GenerateMbTiles.jsx | 183 ------------- src/frontend/src/environment.ts | 16 +- src/frontend/src/views/ProjectDetails.jsx | 5 +- 12 files changed, 418 insertions(+), 298 deletions(-) create mode 100644 src/frontend/src/components/GenerateBasemap.jsx delete mode 100644 src/frontend/src/components/GenerateMbTiles.jsx diff --git a/docker-compose.yml b/docker-compose.yml index cb33a4ad1f..f52994d8bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,7 +97,7 @@ services: entrypoint: ["/migrate-entrypoint.sh"] restart: "on-failure:3" healthcheck: - test: [] # Set the health check test to an empty value to disable it + test: [] # Set the health check test to an empty value to disable it ui: image: "ghcr.io/hotosm/fmtm/frontend:debug" diff --git a/src/backend/app/main.py b/src/backend/app/main.py index beb5d92708..0441bb17f7 100644 --- a/src/backend/app/main.py +++ b/src/backend/app/main.py @@ -71,6 +71,7 @@ def get_application() -> FastAPI: allow_credentials=True, allow_methods=["*"], allow_headers=["*"], + expose_headers=["Content-Disposition"], ) _app.include_router(user_routes.router) diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index cb8611c50d..3864099d3a 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -222,3 +222,4 @@ class BackgroundTaskStatus(IntEnum, Enum): TILES_SOURCE = ["esri", "bing", "google", "topo"] +TILES_FORMATS = ["mbtiles", "sqlitedb", "sqlite3", "sqlite", "pmtiles"] diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 6c15e9a739..ddd7a013df 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -39,7 +39,7 @@ from geoalchemy2.shape import from_shape from geojson import dump from loguru import logger as log -from osm_fieldwork import basemapper +from osm_fieldwork.basemapper import create_basemap_file from osm_fieldwork.data_models import data_models_path from osm_fieldwork.filter_data import FilterData from osm_fieldwork.json2osm import json2osm @@ -2300,19 +2300,25 @@ async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str) def get_project_tiles( db: Session, project_id: int, - source: str, background_task_id: uuid.UUID, + source: str, + output_format: str = "mbtiles", + tms: str = None, ): - """Get the tiles for a project.""" - zooms = [12, 13, 14, 15, 16, 17, 18, 19] - source = source + """Get the tiles for a project. + + Args: + project_id (int): ID of project to create tiles for. + background_task_id (uuid.UUID): UUID of background task to track. + source (str): Tile source ("esri", "bing", "topo", "google", "oam"). + output_format (str, optional): Default "mbtiles". + Other options: "pmtiles", "sqlite3". + tms (str, optional): Default None. Custom TMS provider URL. + """ + zooms = "12-19" tiles_path_id = uuid.uuid4() tiles_dir = f"{TILESDIR}/{tiles_path_id}" - base = f"{tiles_dir}/{source}tiles" - outfile = f"{tiles_dir}/{project_id}_{source}tiles.mbtiles" - - if not os.path.exists(base): - os.makedirs(base) + outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}" tile_path_instance = db_models.DbTilesPath( project_id=project_id, @@ -2327,45 +2333,45 @@ def get_project_tiles( db.commit() # Project Outline + log.debug(f"Getting bbox for project: {project_id}") query = text( - f"""SELECT jsonb_build_object( - 'type', 'FeatureCollection', - 'features', jsonb_agg(feature) - ) - FROM ( - SELECT jsonb_build_object( - 'type', 'Feature', - 'id', id, - 'geometry', ST_AsGeoJSON(outline)::jsonb - ) AS feature - FROM projects - WHERE id={project_id} - ) features;""" + f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon, + ST_YMin(ST_Envelope(outline)) AS min_lat, + ST_XMax(ST_Envelope(outline)) AS max_lon, + ST_YMax(ST_Envelope(outline)) AS max_lat + FROM projects + WHERE id = {project_id};""" ) result = db.execute(query) - features = result.fetchone()[0] - - # Boundary - boundary_file = f"/tmp/{project_id}_boundary.geojson" - - # Update outfile containing osm extracts with the new geojson contents containing title in the properties. - with open(boundary_file, "w") as jsonfile: - jsonfile.truncate(0) - dump(features, jsonfile) + project_bbox = result.fetchone() + log.debug(f"Extracted project bbox: {project_bbox}") - basemap = basemapper.BaseMapper(boundary_file, base, source, False) - outf = basemapper.DataFile(outfile, basemap.getFormat()) - suffix = os.path.splitext(outfile)[1] - if suffix == ".mbtiles": - outf.addBounds(basemap.bbox) - for level in zooms: - basemap.getTiles(level) - if outfile: - # Create output database and specify image format, png, jpg, or tif - outf.writeTiles(basemap.tiles, base) - else: - log.info("Only downloading tiles to %s!" % base) + if project_bbox: + min_lon, min_lat, max_lon, max_lat = project_bbox + else: + log.error(f"Failed to get bbox from project: {project_id}") + + log.debug( + "Creating basemap with params: " + f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | " + f"outfile={outfile} | " + f"zooms={zooms} | " + f"outdir={tiles_dir} | " + f"source={source} | " + f"xy={False} | " + f"tms={tms}" + ) + create_basemap_file( + boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}", + outfile=outfile, + zooms=zooms, + outdir=tiles_dir, + source=source, + xy=False, + tms=tms, + ) + log.info(f"Basemap created for project ID {project_id}: {outfile}") tile_path_instance.status = 4 db.commit() diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index d10c0547d0..b1966bff09 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -18,6 +18,7 @@ import json import os import uuid +from pathlib import Path from typing import List, Optional from fastapi import ( @@ -39,7 +40,7 @@ from ..central import central_crud from ..db import database, db_models -from ..models.enums import TILES_SOURCE +from ..models.enums import TILES_FORMATS, TILES_SOURCE from ..tasks import tasks_crud from . import project_crud, project_schemas, utils from .project_crud import check_crs @@ -977,16 +978,25 @@ async def generate_project_tiles( source: str = Query( ..., description="Select a source for tiles", enum=TILES_SOURCE ), + format: str = Query( + "mbtiles", description="Select an output format", enum=TILES_FORMATS + ), + tms: str = Query( + None, + description="Provide a custom TMS URL, optional", + ), db: Session = Depends(database.get_db), ): - """Returns the tiles for a project. + """Returns basemap tiles for a project. Args: - project_id (int): The id of the project. - source (str): The selected source. + project_id (int): ID of project to create tiles for. + source (str): Tile source ("esri", "bing", "topo", "google", "oam"). + format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3". + tms (str, optional): Default None. Custom TMS provider URL. Returns: - Response: The File response object containing the tiles. + str: Success message that tile generation started. """ # generate a unique task ID using uuid background_task_id = uuid.uuid4() @@ -997,7 +1007,13 @@ async def generate_project_tiles( ) background_tasks.add_task( - project_crud.get_project_tiles, db, project_id, source, background_task_id + project_crud.get_project_tiles, + db, + project_id, + background_task_id, + source, + format, + tms, ) return {"Message": "Tile generation started"} @@ -1018,14 +1034,24 @@ async def tiles_list(project_id: int, db: Session = Depends(database.get_db)): @router.get("/download_tiles/") async def download_tiles(tile_id: int, db: Session = Depends(database.get_db)): + log.debug("Getting tile archive path from DB") tiles_path = ( db.query(db_models.DbTilesPath) .filter(db_models.DbTilesPath.id == str(tile_id)) .first() ) + log.info(f"User requested download for tiles: {tiles_path.path}") + + project_id = tiles_path.project_id + project_name = project_crud.get_project(db, project_id).project_name_prefix + filename = Path(tiles_path.path).name.replace( + f"{project_id}_", f"{project_name.replace(' ', '_')}_" + ) + log.debug(f"Sending tile archive to user: {filename}") + return FileResponse( tiles_path.path, - headers={"Content-Disposition": "attachment; filename=tiles.mbtiles"}, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 7d268d61e7..7888650972 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -6,7 +6,7 @@ groups = ["default", "debug", "dev", "docs", "test"] cross_platform = true static_urls = false lock_version = "4.3" -content_hash = "sha256:65f3117234bb63e9533d5568fb0e6256b66b0dc8a567e6bd7b18c8566df2491f" +content_hash = "sha256:c5ceec798ad38a14fccb76570913b6ce9a8c334b5838d3bef89a4339a1ad2dc6" [[package]] name = "annotated-types" @@ -132,42 +132,42 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.3.0" +version = "3.3.1" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." files = [ - {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, - {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, + {file = "charset-normalizer-3.3.1.tar.gz", hash = "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-win32.whl", hash = "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-win32.whl", hash = "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff"}, + {file = "charset_normalizer-3.3.1-py3-none-any.whl", hash = "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708"}, ] [[package]] @@ -314,18 +314,18 @@ files = [ [[package]] name = "fastapi" -version = "0.103.2" -requires_python = ">=3.7" +version = "0.104.0" +requires_python = ">=3.8" summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" dependencies = [ "anyio<4.0.0,>=3.7.1", "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", "starlette<0.28.0,>=0.27.0", - "typing-extensions>=4.5.0", + "typing-extensions>=4.8.0", ] files = [ - {file = "fastapi-0.103.2-py3-none-any.whl", hash = "sha256:3270de872f0fe9ec809d4bd3d4d890c6d5cc7b9611d721d6438f9dacc8c4ef2e"}, - {file = "fastapi-0.103.2.tar.gz", hash = "sha256:75a11f6bfb8fc4d2bec0bd710c2d5f2829659c0e8c0afd5560fdda6ce25ec653"}, + {file = "fastapi-0.104.0-py3-none-any.whl", hash = "sha256:456482c1178fb7beb2814b88e1885bc49f9a81f079665016feffe3e1c6a7663e"}, + {file = "fastapi-0.104.0.tar.gz", hash = "sha256:9c44de45693ae037b0c6914727a29c49a40668432b67c859a87851fc6a7b74c6"}, ] [[package]] @@ -348,7 +348,7 @@ files = [ [[package]] name = "geoalchemy2" -version = "0.14.1" +version = "0.14.2" requires_python = ">=3.7" summary = "Using SQLAlchemy with Spatial Databases" dependencies = [ @@ -356,8 +356,8 @@ dependencies = [ "packaging", ] files = [ - {file = "GeoAlchemy2-0.14.1-py3-none-any.whl", hash = "sha256:0830c98f83d6b1706e62b5544793d304e2853493d6e70ac18444c13748c3d1c7"}, - {file = "GeoAlchemy2-0.14.1.tar.gz", hash = "sha256:620b31cbf97a368b2486dbcfcd36da2081827e933d4163bcb942043b79b545e8"}, + {file = "GeoAlchemy2-0.14.2-py3-none-any.whl", hash = "sha256:ca81c2d924c0724458102bac93f68f3e3c337a65fcb811af5e504ce7c5d56ac2"}, + {file = "GeoAlchemy2-0.14.2.tar.gz", hash = "sha256:8ca023dcb9a36c6d312f3b4aee631d66385264e2fc9feb0ab0f446eb5609407d"}, ] [[package]] @@ -971,7 +971,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.3.6" +version = "0.3.7" requires_python = ">=3.10" summary = "Processing field data from OpenDataKit to OpenStreetMap format." dependencies = [ @@ -983,9 +983,10 @@ dependencies = [ "haversine>=2.8.0", "levenshtein>=0.21.1", "mercantile>=1.2.1", - "osm-rawdata==0.1.3", + "osm-rawdata>=0.1.3", "overpy>=0.6", "pandas>=2.0.3", + "pmtiles>=3.2.0", "progress>=1.6", "psycopg2>=2.9.7", "py-cpuinfo>=9.0.0", @@ -999,8 +1000,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.3.6.tar.gz", hash = "sha256:f1825cc74b936aebd0baa748b2765f97147abfa8700a4ecf35e2bde4d41cec03"}, - {file = "osm_fieldwork-0.3.6-py3-none-any.whl", hash = "sha256:78056edaa30a9b31004b84a929beda32884bb6ff5d7dbd3a913bef452cdee922"}, + {file = "osm-fieldwork-0.3.7.tar.gz", hash = "sha256:d941d9ba01d93af0eaf0810b72eabb4cac986d07420859593e6302b533a6a9c1"}, + {file = "osm_fieldwork-0.3.7-py3-none-any.whl", hash = "sha256:18ca6dead0e1be63e693da6537b76c077ca9b4e680285a1ff2b2fba44ed6d590"}, ] [[package]] @@ -1020,13 +1021,14 @@ files = [ [[package]] name = "osm-rawdata" -version = "0.1.3" +version = "0.1.4" requires_python = ">=3.10" summary = "Make data extracts from OSM data." dependencies = [ "GeoAlchemy2>=0.12.5", "PyYAML>=6.0.1", "SQLAlchemy-Utils>=0.41.1", + "flatdict>=4.0.1", "geojson>=2.5.0", "psycopg2>=2.9.9", "pyarrow>=1.24.2", @@ -1035,8 +1037,8 @@ dependencies = [ "sqlalchemy>=1.4.41", ] files = [ - {file = "osm-rawdata-0.1.3.tar.gz", hash = "sha256:4ebca28fd1f164ba012bb47e2799aaa211f1799e85b85b2fd73de21090c8c46e"}, - {file = "osm_rawdata-0.1.3-py3-none-any.whl", hash = "sha256:44c2c350a2b51fa491a51d05e9498c056c8cacbdca804dc4d9dec9b7ac0d0d4e"}, + {file = "osm-rawdata-0.1.4.tar.gz", hash = "sha256:f1d44316d932e77c54369c3aa6285322ef8979089981e9f98dc4a1ce6224c9f1"}, + {file = "osm_rawdata-0.1.4-py3-none-any.whl", hash = "sha256:e51694f88367c54cf0220e126e2db94cf86d757a934e42cadc31553daf6afd60"}, ] [[package]] @@ -1155,6 +1157,15 @@ files = [ {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] +[[package]] +name = "pmtiles" +version = "3.2.0" +summary = "Library and utilities to write and read PMTiles files - cloud-optimized archives of map tiles." +files = [ + {file = "pmtiles-3.2.0-py3-none-any.whl", hash = "sha256:f44e85c6622249e99044db7de77e81afa79f12faf3b53f6e0dab7a345f597e8a"}, + {file = "pmtiles-3.2.0.tar.gz", hash = "sha256:49b24af5ba59c505bd70e8459b5eec889c1213cd7fd39eb6132a516eb8dd4301"}, +] + [[package]] name = "pre-commit" version = "3.5.0" @@ -1385,7 +1396,7 @@ files = [ [[package]] name = "pymdown-extensions" -version = "10.3" +version = "10.3.1" requires_python = ">=3.8" summary = "Extension pack for Python Markdown." dependencies = [ @@ -1393,8 +1404,8 @@ dependencies = [ "pyyaml", ] files = [ - {file = "pymdown_extensions-10.3-py3-none-any.whl", hash = "sha256:77a82c621c58a83efc49a389159181d570e370fff9f810d3a4766a75fc678b66"}, - {file = "pymdown_extensions-10.3.tar.gz", hash = "sha256:94a0d8a03246712b64698af223848fd80aaf1ae4c4be29c8c61939b0467b5722"}, + {file = "pymdown_extensions-10.3.1-py3-none-any.whl", hash = "sha256:8cba67beb2a1318cdaf742d09dff7c0fc4cafcc290147ade0f8fb7b71522711a"}, + {file = "pymdown_extensions-10.3.1.tar.gz", hash = "sha256:f6c79941498a458852853872e379e7bab63888361ba20992fc8b4f8a9b61735e"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 7b5ff3f7e4..0f5161f722 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -44,8 +44,8 @@ dependencies = [ "py-cpuinfo>=9.0.0", "loguru>=0.7.0", "osm-login-python==1.0.1", - "osm-fieldwork==0.3.6", - "osm-rawdata==0.1.3", + "osm-fieldwork==0.3.7", + "osm-rawdata==0.1.4", "minio>=7.1.17", "pyproj>=3.6.1", ] diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 66a21d6c87..6386e5df6e 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -162,9 +162,13 @@ export const DownloadTile = (url, payload) => { const response = await CoreModules.axios.get(url, { responseType: 'blob', }); + + // Get filename from content-disposition header + const filename = response.headers['content-disposition'].split('filename=')[1]; + var a = document.createElement('a'); a.href = window.URL.createObjectURL(response.data); - a.download = `${payload.title}_mbtiles.mbtiles`; + a.download = filename; a.click(); dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false })); } catch (error) { diff --git a/src/frontend/src/components/GenerateBasemap.jsx b/src/frontend/src/components/GenerateBasemap.jsx new file mode 100644 index 0000000000..c222c790b1 --- /dev/null +++ b/src/frontend/src/components/GenerateBasemap.jsx @@ -0,0 +1,249 @@ +import React, { useEffect, useState } from 'react'; +import CoreModules from '../shared/CoreModules'; +import AssetModules from '../shared/AssetModules'; +import environment from '../environment'; +import { DownloadTile, GenerateProjectTiles, GetTilesList } from '../api/Project'; + +const GenerateBasemap = ({ setToggleGenerateModal, toggleGenerateModal, projectInfo }) => { + const dispatch = CoreModules.useAppDispatch(); + const params = CoreModules.useParams(); + const encodedId = params.id; + const decodedId = environment.decode(encodedId); + const defaultTheme = CoreModules.useAppSelector((state) => state.theme.hotTheme); + const generateProjectTilesLoading = CoreModules.useAppSelector((state) => state.project.generateProjectTilesLoading); + const tilesList = CoreModules.useAppSelector((state) => state.project.tilesList); + const [selectedTileSource, setSelectedTileSource] = useState(null); + const [selectedOutputFormat, setSelectedOutputFormat] = useState(null); + const [tmsUrl, setTmsUrl] = useState(''); + + const modalStyle = (theme) => ({ + width: '90vw', // Responsive modal width using vw + height: '95vh', + bgcolor: theme.palette.mode === 'dark' ? '#0A1929' : 'white', + border: '1px solid ', + padding: '16px 32px 24px 32px', + }); + const downloadBasemap = (tileId) => { + dispatch(DownloadTile(`${import.meta.env.VITE_API_URL}/projects/download_tiles/?tile_id=${tileId}`, projectInfo)); + }; + + const getTilesList = () => { + dispatch(GetTilesList(`${import.meta.env.VITE_API_URL}/projects/tiles_list/${decodedId}/`)); + }; + + useEffect(() => { + // Only fetch tiles list when the modal is open + if (toggleGenerateModal) { + getTilesList(); + } + }, [toggleGenerateModal]); + + const handleTileSourceChange = (e) => { + setSelectedTileSource(e.target.value); + // If 'tms' is selected, clear the TMS URL + if (e.target.value !== 'tms') { + setTmsUrl(''); + } + }; + + const handleTmsUrlChange = (e) => { + setTmsUrl(e.target.value); + }; + + return ( + setToggleGenerateModal(!toggleGenerateModal)} + > + + {/* Close Button */} + + setToggleGenerateModal(!toggleGenerateModal)} + sx={{ width: '50px', float: 'right', display: 'block' }} + > + + + + + {/* Output Format Dropdown */} + + + + Select Output Format + + { + setSelectedOutputFormat(e.target.value); + }} + > + {environment.tileOutputFormats?.map((form) => ( + + {form.label} + + ))} + + + + + {/* Tile Source Dropdown or TMS URL Input */} + + + + Select Tile Source + + + {environment.baseMapProviders?.map((form) => ( + + {form.label} + + ))} + + {selectedTileSource === 'tms' && ( + + + + )} + + + + {/* Generate Button */} + + { + // Check if 'tms' is selected and tmsUrl is not empty + if (selectedTileSource === 'tms' && !tmsUrl) { + // Handle error, TMS URL is required + console.log('TMS URL is required'); + return; + } + + dispatch( + GenerateProjectTiles( + `${ + import.meta.env.VITE_API_URL + }/projects/tiles/${decodedId}?source=${selectedTileSource}&format=${selectedOutputFormat}&tms=${tmsUrl}`, + decodedId, + ), + ); + }} + > + Generate + + + + {/* Refresh Button */} + + { + getTilesList(); + }} + > + Refresh + + + + {/* Table Content */} + + + + + + Id + Source + Status + + + + + {tilesList.map((list) => ( + + + {list.id} + + {list.tile_source} + + {list.status === 'SUCCESS' ? 'COMPLETED' : list.status} + + + {list.status === 'SUCCESS' ? ( + downloadBasemap(list.id)} + > + ) : ( + <> + )} + + + ))} + + + + + + + ); +}; + +GenerateBasemap.propTypes = {}; + +export default GenerateBasemap; diff --git a/src/frontend/src/components/GenerateMbTiles.jsx b/src/frontend/src/components/GenerateMbTiles.jsx deleted file mode 100644 index 09060212a3..0000000000 --- a/src/frontend/src/components/GenerateMbTiles.jsx +++ /dev/null @@ -1,183 +0,0 @@ -import React, { useEffect, useState } from 'react'; -import CoreModules from '../shared/CoreModules'; -import AssetModules from '../shared/AssetModules'; -import environment from '../environment'; -import { DownloadTile, GenerateProjectTiles, GetTilesList } from '../api/Project'; - -const GenerateMbTiles = ({ setToggleGenerateModal, toggleGenerateModal, projectInfo }) => { - const dispatch = CoreModules.useAppDispatch(); - const params = CoreModules.useParams(); - const encodedId = params.id; - const decodedId = environment.decode(encodedId); - const defaultTheme = CoreModules.useAppSelector((state) => state.theme.hotTheme); - const generateProjectTilesLoading = CoreModules.useAppSelector((state) => state.project.generateProjectTilesLoading); - const tilesList = CoreModules.useAppSelector((state) => state.project.tilesList); - const [selectedTileSource, setSelectedTileSource] = useState(null); - - const modalStyle = (theme) => ({ - // width: "30%", - // height: "24%", - bgcolor: theme.palette.mode === 'dark' ? '#0A1929' : 'white', - border: '1px solid ', - padding: '16px 32px 24px 32px', - }); - const downloadMbTiles = (tileId) => { - dispatch(DownloadTile(`${import.meta.env.VITE_API_URL}/projects/download_tiles/?tile_id=${tileId}`, projectInfo)); - }; - const getTilesList = () => { - dispatch(GetTilesList(`${import.meta.env.VITE_API_URL}/projects/tiles_list/${decodedId}/`)); - }; - useEffect(() => { - //Only fetch tiles list when modal is open - if (toggleGenerateModal) { - getTilesList(); - } - }, [toggleGenerateModal]); - return ( - setToggleGenerateModal(!toggleGenerateModal)} - > - <> - setToggleGenerateModal(!toggleGenerateModal)} - sx={{ width: '50px', float: 'right', display: 'block' }} - > - - - - - Select Tiles Source - - { - setSelectedTileSource(e.target.value); - // handleCustomChange("form_ways", e.target.value); - }} - // onChange={(e) => dispatch(CreateProjectActions.SetProjectDetails({ key: 'form_ways', value: e.target.value }))} - > - {environment.selectFormWays?.map((form) => ( - - {form.label} - - ))} - - {/* {errors.form_ways && ( - - {errors.form_ways} - - )} */} - -
- { - // setToggleGenerateModal(false); - dispatch( - GenerateProjectTiles( - `${import.meta.env.VITE_API_URL}/projects/tiles/${decodedId}?source=${selectedTileSource}`, - decodedId, - ), - ); - // dispatch(CoreModules.TaskActions.SetJosmEditorError(null)); - }} - > - Generate - -
-
- { - getTilesList(); - }} - > - Refresh - -
- - - - - Id - Source - Status - - - - - {tilesList.map((list) => ( - - - {list.id} - - {list.tile_source} - - {/* Changed Success Display to Completed */} - {list.status === 'SUCCESS' ? 'COMPLETED' : list.status} - - - {list.status === 'SUCCESS' ? ( - downloadMbTiles(list.id)} - > - ) : ( - <> - )} - - - ))} - - - - - - ); -}; - -GenerateMbTiles.propTypes = {}; - -export default GenerateMbTiles; diff --git a/src/frontend/src/environment.ts b/src/frontend/src/environment.ts index 38f9de9f28..b4a4e02954 100755 --- a/src/frontend/src/environment.ts +++ b/src/frontend/src/environment.ts @@ -40,11 +40,17 @@ export default { // "SPLIT", // "ARCHIVED", ], - selectFormWays: [ - { id: 1, label: 'esri', value: 'esri' }, - { id: 2, label: 'bing', value: 'bing' }, - { id: 3, label: 'google', value: 'google' }, - { id: 4, label: 'topo', value: 'topo' }, + baseMapProviders: [ + { id: 1, label: 'ESRI', value: 'esri' }, + { id: 2, label: 'Bing', value: 'bing' }, + { id: 3, label: 'Google', value: 'google' }, + { id: 4, label: 'Topo', value: 'topo' }, + { id: 5, label: 'Custom TMS', value: 'tms' }, + ], + tileOutputFormats: [ + { id: 1, label: 'MBTiles', value: 'mbtiles' }, + { id: 2, label: 'OSMAnd', value: 'sqlite3' }, + { id: 3, label: 'PMTiles', value: 'pmtiles' }, ], statusColors: { PENDING: 'gray', diff --git a/src/frontend/src/views/ProjectDetails.jsx b/src/frontend/src/views/ProjectDetails.jsx index ae2960eb34..cf9bbcc999 100755 --- a/src/frontend/src/views/ProjectDetails.jsx +++ b/src/frontend/src/views/ProjectDetails.jsx @@ -24,8 +24,7 @@ import AssetModules from '../shared/AssetModules'; import GeoJSON from 'ol/format/GeoJSON'; import FmtmLogo from '../assets/images/hotLog.png'; -import Overlay from 'ol/Overlay'; -import GenerateMbTiles from '../components/GenerateMbTiles'; +import GenerateBasemap from '../components/GenerateBasemap'; import { ProjectBuildingGeojsonService } from '../api/SubmissionService'; import { get } from 'ol/proj'; import { buildingStyle, basicGeojsonTemplate } from '../utilities/mapUtils'; @@ -287,7 +286,7 @@ const Home = () => {
{/* Customized Modal For Generate Tiles */}
-