From e9400ab5b7fb7d07dd2e4d25c0cde27ecd10587e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 9 Feb 2023 17:54:15 +0100 Subject: [PATCH 01/10] sketchout XarrayTiler --- pctiler/pctiler/endpoints/zarr.py | 225 ++++++++++++++++++++++++++++++ pctiler/setup.py | 4 - 2 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 pctiler/pctiler/endpoints/zarr.py diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py new file mode 100644 index 00000000..27a555d9 --- /dev/null +++ b/pctiler/pctiler/endpoints/zarr.py @@ -0,0 +1,225 @@ +from dataclasses import dataclass +from typing import List, Literal, Optional, Tuple, Type +from urllib.parse import urlencode + +import xarray +from fastapi import Depends, Path, Query +from rio_tiler.io import BaseReader, XarrayReader +from rio_tiler.models import Info +from starlette.requests import Request +from starlette.responses import Response +from titiler.core.dependencies import RescalingParams +from titiler.core.factory import BaseTilerFactory, img_endpoint_params +from titiler.core.models.mapbox import TileJSON +from titiler.core.resources.enums import ImageType +from titiler.core.resources.responses import JSONResponse + + +@dataclass +class XarrayTilerFactory(BaseTilerFactory): + + # Default reader is set to rio_tiler.io.Reader + reader: Type[BaseReader] = XarrayReader + + def register_routes(self): + """Register Info / Tiles / TileJSON endoints.""" + + @self.router.get( + "/info", + response_model=Info, + response_model_exclude_none=True, + response_class=JSONResponse, + responses={200: {"description": "Return dataset's basic info."}}, + ) + def info_endpoint( + src_path=Depends(self.path_dependency), + variable: str = Query(..., description="Xarray Variable"), + ): + """Return dataset's basic info.""" + with xarray.open_dataset( + src_path, engine="zarr", decode_coords="all" + ) as src: + ds = src[variable][:1] + + # Make sure we are a CRS + crs = ds.rio.crs or "epsg:4326" + ds.rio.write_crs(crs, inplace=True) + + with self.reader(ds) as dst: + return dst.info() + + @self.router.get(r"/tiles/{z}/{x}/{y}", **img_endpoint_params) + @self.router.get(r"/tiles/{z}/{x}/{y}.{format}", **img_endpoint_params) + @self.router.get(r"/tiles/{z}/{x}/{y}@{scale}x", **img_endpoint_params) + @self.router.get(r"/tiles/{z}/{x}/{y}@{scale}x.{format}", **img_endpoint_params) + @self.router.get(r"/tiles/{TileMatrixSetId}/{z}/{x}/{y}", **img_endpoint_params) + @self.router.get( + r"/tiles/{TileMatrixSetId}/{z}/{x}/{y}.{format}", **img_endpoint_params + ) + @self.router.get( + r"/tiles/{TileMatrixSetId}/{z}/{x}/{y}@{scale}x", **img_endpoint_params + ) + @self.router.get( + r"/tiles/{TileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + **img_endpoint_params, + ) + def tiles_endpoint( + z: int = Path(..., ge=0, le=30, description="TileMatrixSet zoom level"), + x: int = Path(..., description="TileMatrixSet column"), + y: int = Path(..., description="TileMatrixSet row"), + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( + self.default_tms, + description=f"TileMatrixSet Name (default: '{self.default_tms}')", + ), + scale: int = Query( + 1, gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..." + ), + format: ImageType = Query( + None, description="Output image type. Default is auto." + ), + src_path=Depends(self.path_dependency), + variable: str = Query(..., description="Xarray Variable"), + post_process=Depends(self.process_dependency), + rescale: Optional[List[Tuple[float, ...]]] = Depends(RescalingParams), + color_formula: Optional[str] = Query( + None, + title="Color Formula", + description=( + "rio-color formula (info: https://github.com/mapbox/rio-color)" + ), + ), + colormap=Depends(self.colormap_dependency), + render_params=Depends(self.render_dependency), + ): + """Create map tile from a dataset.""" + tms = self.supported_tms.get(TileMatrixSetId) + + with xarray.open_dataset( + src_path, engine="zarr", decode_coords="all" + ) as src: + ds = src[variable][:1] + + # Make sure we are a CRS + crs = ds.rio.crs or "epsg:4326" + ds.rio.write_crs(crs, inplace=True) + + with self.reader(ds, tms=tms) as dst: + image = dst.tile( + x, + y, + z, + tilesize=scale * 256, + ) + + if post_process: + image = post_process(image) + + if rescale: + image.rescale(rescale) + + if color_formula: + image.apply_color_formula(color_formula) + + if not format: + format = ImageType.jpeg if image.mask.all() else ImageType.png + + content = image.render( + img_format=format.driver, + colormap=colormap, + **format.profile, + **render_params, + ) + + return Response(content, media_type=format.mediatype) + + @self.router.get( + "/tilejson.json", + response_model=TileJSON, + responses={200: {"description": "Return a tilejson"}}, + response_model_exclude_none=True, + ) + @self.router.get( + "/{TileMatrixSetId}/tilejson.json", + response_model=TileJSON, + responses={200: {"description": "Return a tilejson"}}, + response_model_exclude_none=True, + ) + def tilejson_endpoint( + request: Request, + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( + self.default_tms, + description=f"TileMatrixSet Name (default: '{self.default_tms}')", + ), + src_path=Depends(self.path_dependency), + variable: str = Query(..., description="Xarray Variable"), + tile_format: Optional[ImageType] = Query( + None, description="Output image type. Default is auto." + ), + tile_scale: int = Query( + 1, gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..." + ), + minzoom: Optional[int] = Query( + None, description="Overwrite default minzoom." + ), + maxzoom: Optional[int] = Query( + None, description="Overwrite default maxzoom." + ), + post_process=Depends(self.process_dependency), # noqa + rescale: Optional[List[Tuple[float, ...]]] = Depends( + RescalingParams + ), # noqa + color_formula: Optional[str] = Query( # noqa + None, + title="Color Formula", + description=( + "rio-color formula (info: https://github.com/mapbox/rio-color)" + ), + ), + colormap=Depends(self.colormap_dependency), # noqa + render_params=Depends(self.render_dependency), # noqa + ): + """Return TileJSON document for a dataset.""" + route_params = { + "z": "{z}", + "x": "{x}", + "y": "{y}", + "scale": tile_scale, + "TileMatrixSetId": TileMatrixSetId, + } + if tile_format: + route_params["format"] = tile_format.value + tiles_url = self.url_for(request, "tiles_endpoint", **route_params) + + qs_key_to_remove = [ + "tilematrixsetid", + "tile_format", + "tile_scale", + "minzoom", + "maxzoom", + ] + qs = [ + (key, value) + for (key, value) in request.query_params._list + if key.lower() not in qs_key_to_remove + ] + if qs: + tiles_url += f"?{urlencode(qs)}" + + tms = self.supported_tms.get(TileMatrixSetId) + + with xarray.open_dataset( + src_path, engine="zarr", decode_coords="all" + ) as src: + ds = src[variable][:1] + + # Make sure we are a CRS + crs = ds.rio.crs or "epsg:4326" + ds.rio.write_crs(crs, inplace=True) + + with self.reader(ds, tms=tms) as src_dst: + return { + "bounds": src_dst.geographic_bounds, + "minzoom": minzoom if minzoom is not None else src_dst.minzoom, + "maxzoom": maxzoom if maxzoom is not None else src_dst.maxzoom, + "tiles": [tiles_url], + } diff --git a/pctiler/setup.py b/pctiler/setup.py index 3626d5ed..be5a8250 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -8,18 +8,14 @@ "jinja2==3.0.3", "pystac==1.*", "planetary-computer==0.4.*", - "rasterio==1.3.*", "titiler.core==0.10.2", "titiler.mosaic==0.10.2", - # titiler-pgstac "psycopg[binary,pool]", "titiler.pgstac==0.2.2", - # colormap dependencies "matplotlib==3.4.*", - "importlib_resources>=1.1.0;python_version<'3.9'", "pccommon", ] From 28d66b8921d2b888966dd5d8c4c2dd4ae6671be8 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 9 Feb 2023 17:55:41 +0100 Subject: [PATCH 02/10] add missing requirements --- pctiler/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pctiler/setup.py b/pctiler/setup.py index be5a8250..8c090ee1 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -18,6 +18,8 @@ "matplotlib==3.4.*", "importlib_resources>=1.1.0;python_version<'3.9'", "pccommon", + "xarray", + "rioxarray", ] extra_reqs = { From 4f1c6c16fb289b99c915808bb634bb95e4212630 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 28 Feb 2023 15:02:08 +0100 Subject: [PATCH 03/10] fix mypy issues --- pctiler/pctiler/endpoints/zarr.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py index 27a555d9..1536a339 100644 --- a/pctiler/pctiler/endpoints/zarr.py +++ b/pctiler/pctiler/endpoints/zarr.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Literal, Optional, Tuple, Type +from typing import Dict, List, Literal, Optional, Tuple, Type from urllib.parse import urlencode import xarray @@ -21,7 +21,7 @@ class XarrayTilerFactory(BaseTilerFactory): # Default reader is set to rio_tiler.io.Reader reader: Type[BaseReader] = XarrayReader - def register_routes(self): + def register_routes(self) -> None: # type: ignore """Register Info / Tiles / TileJSON endoints.""" @self.router.get( @@ -32,9 +32,9 @@ def register_routes(self): responses={200: {"description": "Return dataset's basic info."}}, ) def info_endpoint( - src_path=Depends(self.path_dependency), + src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), - ): + ) -> Info: """Return dataset's basic info.""" with xarray.open_dataset( src_path, engine="zarr", decode_coords="all" @@ -63,11 +63,11 @@ def info_endpoint( r"/tiles/{TileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", **img_endpoint_params, ) - def tiles_endpoint( + def tiles_endpoint( # type: ignore z: int = Path(..., ge=0, le=30, description="TileMatrixSet zoom level"), x: int = Path(..., description="TileMatrixSet column"), y: int = Path(..., description="TileMatrixSet row"), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( # type: ignore self.default_tms, description=f"TileMatrixSet Name (default: '{self.default_tms}')", ), @@ -77,7 +77,7 @@ def tiles_endpoint( format: ImageType = Query( None, description="Output image type. Default is auto." ), - src_path=Depends(self.path_dependency), + src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), post_process=Depends(self.process_dependency), rescale: Optional[List[Tuple[float, ...]]] = Depends(RescalingParams), @@ -90,7 +90,7 @@ def tiles_endpoint( ), colormap=Depends(self.colormap_dependency), render_params=Depends(self.render_dependency), - ): + ) -> Response: """Create map tile from a dataset.""" tms = self.supported_tms.get(TileMatrixSetId) @@ -144,13 +144,13 @@ def tiles_endpoint( responses={200: {"description": "Return a tilejson"}}, response_model_exclude_none=True, ) - def tilejson_endpoint( + def tilejson_endpoint( # type: ignore request: Request, - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( # type: ignore self.default_tms, description=f"TileMatrixSet Name (default: '{self.default_tms}')", ), - src_path=Depends(self.path_dependency), + src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), tile_format: Optional[ImageType] = Query( None, description="Output image type. Default is auto." @@ -177,7 +177,7 @@ def tilejson_endpoint( ), colormap=Depends(self.colormap_dependency), # noqa render_params=Depends(self.render_dependency), # noqa - ): + ) -> Dict: """Return TileJSON document for a dataset.""" route_params = { "z": "{z}", From 02fdfb400413f7d6ef13b30376c362f3a3bf12d4 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 28 Feb 2023 16:09:30 +0100 Subject: [PATCH 04/10] add time slice --- pctiler/pctiler/endpoints/zarr.py | 47 ++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py index 1536a339..479f9b39 100644 --- a/pctiler/pctiler/endpoints/zarr.py +++ b/pctiler/pctiler/endpoints/zarr.py @@ -24,6 +24,19 @@ class XarrayTilerFactory(BaseTilerFactory): def register_routes(self) -> None: # type: ignore """Register Info / Tiles / TileJSON endoints.""" + @self.router.get( + "/variables", + response_class=JSONResponse, + responses={200: {"description": "Return dataset's Variables."}}, + ) + def variable_endpoint( + src_path: str = Depends(self.path_dependency), + ) -> List[str]: + with xarray.open_dataset( + src_path, engine="zarr", decode_coords="all" + ) as src: + return [i for i in src.data_vars] # type: ignore + @self.router.get( "/info", response_model=Info, @@ -34,19 +47,36 @@ def register_routes(self) -> None: # type: ignore def info_endpoint( src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), + show_times: bool = Query( + None, description="Show info about the time dimension" + ), ) -> Info: """Return dataset's basic info.""" + show_times = show_times or False + with xarray.open_dataset( src_path, engine="zarr", decode_coords="all" ) as src: - ds = src[variable][:1] + ds = src[variable] + times = [] + if "time" in ds.dims: + times = [str(x.data) for x in ds.time] + # To avoid returning huge a `band_metadata` and `band_descriptions` + # we only return info of the first time slice + ds = src[variable][0] # Make sure we are a CRS crs = ds.rio.crs or "epsg:4326" ds.rio.write_crs(crs, inplace=True) with self.reader(ds) as dst: - return dst.info() + info = dst.info().dict() + + if times and show_times: + info["count"] = len(times) + info["times"] = times + + return info @self.router.get(r"/tiles/{z}/{x}/{y}", **img_endpoint_params) @self.router.get(r"/tiles/{z}/{x}/{y}.{format}", **img_endpoint_params) @@ -79,6 +109,9 @@ def tiles_endpoint( # type: ignore ), src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), + time_slice: int = Query( + None, description="Slice of time to read (if available)" + ), post_process=Depends(self.process_dependency), rescale: Optional[List[Tuple[float, ...]]] = Depends(RescalingParams), color_formula: Optional[str] = Query( @@ -97,7 +130,10 @@ def tiles_endpoint( # type: ignore with xarray.open_dataset( src_path, engine="zarr", decode_coords="all" ) as src: - ds = src[variable][:1] + ds = src[variable] + if "time" in ds.dims: + time_slice = time_slice or 0 + ds = ds[time_slice : time_slice + 1] # Make sure we are a CRS crs = ds.rio.crs or "epsg:4326" @@ -152,6 +188,9 @@ def tilejson_endpoint( # type: ignore ), src_path: str = Depends(self.path_dependency), variable: str = Query(..., description="Xarray Variable"), + time_slice: int = Query( + None, description="Slice of time to read (if available)" + ), # noqa tile_format: Optional[ImageType] = Query( None, description="Output image type. Default is auto." ), @@ -210,7 +249,7 @@ def tilejson_endpoint( # type: ignore with xarray.open_dataset( src_path, engine="zarr", decode_coords="all" ) as src: - ds = src[variable][:1] + ds = src[variable] # Make sure we are a CRS crs = ds.rio.crs or "epsg:4326" From d1948f6b1f5ff56f522d005a53787406d11ea97f Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 28 Feb 2023 17:28:08 +0100 Subject: [PATCH 05/10] add /zarr endpoints --- pctiler/pctiler/config.py | 1 + pctiler/pctiler/endpoints/zarr.py | 9 +++++++++ pctiler/pctiler/main.py | 8 +++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pctiler/pctiler/config.py b/pctiler/pctiler/config.py index d278758e..1093f2f2 100644 --- a/pctiler/pctiler/config.py +++ b/pctiler/pctiler/config.py @@ -37,6 +37,7 @@ class Settings(BaseSettings): item_endpoint_prefix: str = "/item" mosaic_endpoint_prefix: str = "/mosaic" legend_endpoint_prefix: str = "/legend" + zarr_endpoint_prefix: str = "/zarr" vector_tile_endpoint_prefix: str = "/vector" vector_tile_sa_base_url: str = Field(env=VECTORTILE_SA_BASE_URL_ENV_VAR) diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py index 479f9b39..794dc596 100644 --- a/pctiler/pctiler/endpoints/zarr.py +++ b/pctiler/pctiler/endpoints/zarr.py @@ -14,6 +14,9 @@ from titiler.core.resources.enums import ImageType from titiler.core.resources.responses import JSONResponse +from pctiler.colormaps import PCColorMapParams +from pctiler.config import get_settings + @dataclass class XarrayTilerFactory(BaseTilerFactory): @@ -262,3 +265,9 @@ def tilejson_endpoint( # type: ignore "maxzoom": maxzoom if maxzoom is not None else src_dst.maxzoom, "tiles": [tiles_url], } + + +zarr_factory = XarrayTilerFactory( + colormap_dependency=PCColorMapParams, + router_prefix=get_settings().zarr_endpoint_prefix, +) diff --git a/pctiler/pctiler/main.py b/pctiler/pctiler/main.py index ebb07b28..839b7155 100755 --- a/pctiler/pctiler/main.py +++ b/pctiler/pctiler/main.py @@ -26,7 +26,7 @@ ) from pccommon.openapi import fixup_schema from pctiler.config import get_settings -from pctiler.endpoints import health, item, legend, pg_mosaic, vector_tiles +from pctiler.endpoints import health, item, legend, pg_mosaic, vector_tiles, zarr # Initialize logging init_logging(ServiceName.TILER) @@ -73,6 +73,12 @@ tags=["Collection vector tile endpoints"], ) +app.include_router( + zarr.zarr_factory.router, + prefix=settings.zarr_endpoint_prefix, + tags=["Preview"], +) + app.include_router(health.health_router, tags=["Liveliness/Readiness"]) app.add_middleware(RequestTracingMiddleware, service_name=ServiceName.TILER) From 00b0f74f6dc9c40304bc6d5722a74573852fd678 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 28 Feb 2023 20:47:14 +0100 Subject: [PATCH 06/10] ignore type --- pctiler/pctiler/endpoints/zarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py index 794dc596..4110613d 100644 --- a/pctiler/pctiler/endpoints/zarr.py +++ b/pctiler/pctiler/endpoints/zarr.py @@ -268,6 +268,6 @@ def tilejson_endpoint( # type: ignore zarr_factory = XarrayTilerFactory( - colormap_dependency=PCColorMapParams, - router_prefix=get_settings().zarr_endpoint_prefix, + colormap_dependency=PCColorMapParams, # type: ignore + router_prefix=get_settings().zarr_endpoint_prefix, # type: ignore ) From c19906eeacd1846bff0bc7388e747af9e242c0f5 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 28 Feb 2023 21:15:17 +0100 Subject: [PATCH 07/10] fix flake8 issues --- pctiler/pctiler/endpoints/zarr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pctiler/pctiler/endpoints/zarr.py b/pctiler/pctiler/endpoints/zarr.py index 4110613d..d4012225 100644 --- a/pctiler/pctiler/endpoints/zarr.py +++ b/pctiler/pctiler/endpoints/zarr.py @@ -100,7 +100,9 @@ def tiles_endpoint( # type: ignore z: int = Path(..., ge=0, le=30, description="TileMatrixSet zoom level"), x: int = Path(..., description="TileMatrixSet column"), y: int = Path(..., description="TileMatrixSet row"), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( # type: ignore + TileMatrixSetId: Literal[ # type: ignore + tuple(self.supported_tms.list()) + ] = Query( self.default_tms, description=f"TileMatrixSet Name (default: '{self.default_tms}')", ), @@ -185,7 +187,9 @@ def tiles_endpoint( # type: ignore ) def tilejson_endpoint( # type: ignore request: Request, - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( # type: ignore + TileMatrixSetId: Literal[ # type: ignore + tuple(self.supported_tms.list()) + ] = Query( self.default_tms, description=f"TileMatrixSet Name (default: '{self.default_tms}')", ), From 53fbfe60909e414a952e6c596b3e1dc309cbdb03 Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Wed, 8 Mar 2023 22:40:21 +0100 Subject: [PATCH 08/10] Update pctiler/setup.py --- pctiler/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pctiler/setup.py b/pctiler/setup.py index 8c090ee1..d8577e87 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -20,6 +20,7 @@ "pccommon", "xarray", "rioxarray", + "zarr", ] extra_reqs = { From f5fca3ad441e152fd857da90051fccddc167a37b Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Wed, 8 Mar 2023 23:01:52 +0100 Subject: [PATCH 09/10] Update pctiler/setup.py --- pctiler/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pctiler/setup.py b/pctiler/setup.py index d8577e87..1a2541b6 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -21,6 +21,7 @@ "xarray", "rioxarray", "zarr", + "fsspec", ] extra_reqs = { From 80d7d2b26f7c2d4957b7a1d67084a8fc3fff653b Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 20 Mar 2023 09:44:06 +0100 Subject: [PATCH 10/10] add http/azure fsspec dependencies --- pctiler/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pctiler/setup.py b/pctiler/setup.py index 1a2541b6..a86a9fd4 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -22,6 +22,9 @@ "rioxarray", "zarr", "fsspec", + "requests", + "aiohttp", + "adlfs", ] extra_reqs = {