Skip to content

Commit

Permalink
refactor: allow generating basemaps by user
Browse files Browse the repository at this point in the history
  • Loading branch information
@spwoodcock
spwoodcock committed Jan 8, 2024
1 parent 92fba46 commit c203b4a
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 118 deletions.
132 changes: 33 additions & 99 deletions src/backend/app/projects/project_crud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,17 @@
from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson, timestamp
from app.organization import organization_crud
from app.projects import project_schemas
from app.s3 import add_obj_to_bucket, get_obj_from_bucket
from app.s3 import (
add_file_to_bucket,
add_obj_to_bucket,
get_bucket_path,
get_obj_from_bucket,
)
from app.tasks import tasks_crud
from app.users import user_crud

QR_CODES_DIR = "QR_codes/"
TASK_GEOJSON_DIR = "geojson/"
TILESDIR = "/opt/tiles"


async def get_projects(
Expand Down Expand Up @@ -2122,10 +2126,19 @@ def get_project_tiles(
project_id: int,
background_task_id: uuid.UUID,
source: str,

# NOTE defined as non-async to run in separate thread
def generate_basemap_for_bbox(
db: Session,
project_id: int,
bbox: tuple,
background_task_id: uuid.UUID,
source: str,
output_format: str = "mbtiles",
tms: str = None,
task_id: int = None,
):
"""Get the tiles for a project.
"""Get basemap tiles for a project.
Args:
db (Session): SQLAlchemy db session.
Expand All @@ -2135,11 +2148,11 @@ def get_project_tiles(
output_format (str, optional): Default "mbtiles".
Other options: "pmtiles", "sqlite3".
tms (str, optional): Default None. Custom TMS provider URL.
task_id (bool): If set, create for a task boundary only.
"""
zooms = "12-19"
tiles_path_id = uuid.uuid4()
tiles_dir = f"{TILESDIR}/{tiles_path_id}"
outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
tiles_dir = "opt/tiles"
outfile = f"/tmp/{project_id}_{uuid.uuid4()}.{output_format}"

tile_path_instance = db_models.DbTilesPath(
project_id=project_id,
Expand All @@ -2153,36 +2166,9 @@ def get_project_tiles(
db.add(tile_path_instance)
db.commit()

# Project Outline
log.debug(f"Getting bbox for project: {project_id}")
query = text(
f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
ST_YMin(ST_Envelope(outline)) AS min_lat,
ST_XMax(ST_Envelope(outline)) AS max_lon,
ST_YMax(ST_Envelope(outline)) AS max_lat
FROM projects
WHERE id = {project_id};"""
)

result = db.execute(query)
project_bbox = result.fetchone()
log.debug(f"Extracted project bbox: {project_bbox}")
# Get coords from bbox
min_lon, min_lat, max_lon, max_lat = bbox

if project_bbox:
min_lon, min_lat, max_lon, max_lat = project_bbox
else:
log.error(f"Failed to get bbox from project: {project_id}")

log.debug(
"Creating basemap with params: "
f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
f"outfile={outfile} | "
f"zooms={zooms} | "
f"outdir={tiles_dir} | "
f"source={source} | "
f"xy={False} | "
f"tms={tms}"
)
create_basemap_file(
boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
outfile=outfile,
Expand All @@ -2194,77 +2180,24 @@ def get_project_tiles(
)
log.info(f"Basemap created for project ID {project_id}: {outfile}")

get_bucket_path_sync = async_to_sync(get_bucket_path)
project_s3_path = get_bucket_path_sync(db, project_id)

get_project_sync = async_to_sync(get_project)
project = get_project_sync(db, project_id)
if task_id:
s3_tile_path = f"{project_s3_path}/basemaps/{task_id}.{output_format}"
else:
s3_tile_path = f"{project_s3_path}/basemap.{output_format}"

from app.s3 import add_file_to_bucket
add_file_to_bucket(
settings.S3_BUCKET_NAME,
f"/{project.organisation_id}/{project_id}/basemap.mbtiles",
outfile
s3_tile_path,
outfile,
)

# Generate mbtiles for each task
get_tasks_async = async_to_sync(tasks_crud.get_task_id_list)
task_list = get_tasks_async(db, project_id)

for task_id in task_list:
try:
log.debug(f"Getting bbox for task: {task_id}")
query = text(
f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
ST_YMin(ST_Envelope(outline)) AS min_lat,
ST_XMax(ST_Envelope(outline)) AS max_lon,
ST_YMax(ST_Envelope(outline)) AS max_lat
FROM tasks
WHERE id = {task_id};"""
)

result = db.execute(query)
task_bbox = result.fetchone()
log.debug(f"Extracted task bbox: {task_bbox}")

if task_bbox:
min_lon, min_lat, max_lon, max_lat = task_bbox
else:
log.error(f"Failed to get bbox from task: {project_id}")

task_basemap_outfile = f"{tiles_dir}/{task_id}_{source}tiles.{output_format}"

log.debug(
"Creating basemap with params: "
f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
f"outfile={task_basemap_outfile} | "
f"zooms={zooms} | "
f"outdir={tiles_dir} | "
f"source={source} | "
f"xy={False} | "
f"tms={tms}"
)
create_basemap_file(
boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
outfile=task_basemap_outfile,
zooms=zooms,
outdir=tiles_dir,
source=source,
xy=False,
tms=tms,
)
log.info(f"Basemap created for task ID {task_id}: {task_basemap_outfile}")
except Exception as e:
log.error(str(e))
continue

# Upload task mbtile to s3
add_file_to_bucket(
settings.S3_BUCKET_NAME,
f"/{project.organisation_id}/{project_id}/basemap/{task_id}.mbtiles",
task_basemap_outfile
)


tile_path_instance.status = 4
tile_path_instance.path = (
f"{settings.S3_DOWNLOAD_ROOT}/" f"{settings.S3_BUCKET_NAME}{s3_tile_path}"
)
db.commit()

# Update background task status to COMPLETED
Expand All @@ -2278,6 +2211,7 @@ def get_project_tiles(
log.error(f"Tiles generation process failed for project id {project_id}")

tile_path_instance.status = 2
tile_path_instance.path = ""
db.commit()

# Update background task status to FAILED
Expand Down
44 changes: 25 additions & 19 deletions src/backend/app/projects/project_routes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import json
import os
import uuid
from pathlib import Path
from typing import Optional

import geojson
Expand All @@ -35,21 +34,20 @@
Response,
UploadFile,
)
from fastapi.responses import FileResponse, JSONResponse
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from loguru import logger as log
from osm_fieldwork.make_data_extract import getChoices
from osm_fieldwork.xlsforms import xlsforms_path
from sqlalchemy.orm import Session

from app.auth.osm import AuthUser, login_required
from app.central import central_crud
from app.db import database, db_models
from app.models.enums import TILES_FORMATS, TILES_SOURCE
from app.projects import project_crud, project_schemas
from app.projects.project_crud import check_crs
from app.submission import submission_crud

from ..central import central_crud
from ..db import database, db_models
from ..models.enums import TILES_FORMATS, TILES_SOURCE
from ..tasks import tasks_crud
from . import project_crud, project_schemas
from .project_crud import check_crs
from app.tasks import tasks_crud

router = APIRouter(
prefix="/projects",
Expand Down Expand Up @@ -990,6 +988,10 @@ async def download_features(project_id: int, db: Session = Depends(database.get_
async def generate_project_tiles(
background_tasks: BackgroundTasks,
project_id: int,
task_id: str = Query(
None,
description="Optional task id to generate for",
),
source: str = Query(
..., description="Select a source for tiles", enum=TILES_SOURCE
),
Expand All @@ -1006,12 +1008,13 @@ async def generate_project_tiles(
Args:
project_id (int): ID of project to create tiles for.
task_id (int): Optional task ID (task area) to generate for.
source (str): Tile source ("esri", "bing", "topo", "google", "oam").
format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3".
tms (str, optional): Default None. Custom TMS provider URL.
Returns:
str: Success message that tile generation started.
dict: Success message that tile generation started.
"""
# Create task in db and return uuid
log.debug(
Expand All @@ -1022,18 +1025,19 @@ async def generate_project_tiles(
db, name="generate tiles", project_id=project_id
)


background_tasks.add_task(
project_crud.get_project_tiles,
project_crud.generate_project_or_task_basemap,
db,
project_id,
background_task_id,
source,
format,
tms,
task_id,
)

return {"Message": "Tile generation started"}
return JSONResponse(status_code=200, content={"success": True})



@router.get("/tiles_list/{project_id}/")
Expand Down Expand Up @@ -1173,14 +1177,15 @@ async def get_template_file(
)


@router.get("/project_dashboard/{project_id}", response_model=project_schemas.ProjectDashboard)
@router.get(
"/project_dashboard/{project_id}", response_model=project_schemas.ProjectDashboard
)
async def project_dashboard(
project_id: int,
project_id: int,
background_tasks: BackgroundTasks,
db: Session = Depends(database.get_db)
db: Session = Depends(database.get_db),
):
"""
Get the project dashboard details.
"""Get the project dashboard details.
Args:
project_id (int): The ID of the project.
Expand All @@ -1199,6 +1204,7 @@ async def project_dashboard(
)
return data


@router.get("/contributors/{project_id}")
async def get_contributors(project_id: int, db: Session = Depends(database.get_db)):
"""Get contributors of a project.
Expand All @@ -1210,4 +1216,4 @@ async def get_contributors(project_id: int, db: Session = Depends(database.get_d
list[project_schemas.ProjectUser]: List of project users.
"""
project_users = await project_crud.get_project_users(db, project_id)
return project_users
return project_users

0 comments on commit c203b4a

Please sign in to comment.