Add basic cog support

Cargo.toml

@@ -59,6 +59,7 @@ moka = { version = "0.12", features = ["future"] }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.1", features = ["freetype"] }
 pmtiles = { version = "0.11", features = ["http-async", "mmap-async-tokio", "tilejson", "reqwest-rustls-tls-native-roots"] }
+png = "0.17.14"
 postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-protocol = "0.6"
@@ -85,6 +86,7 @@ static-files = "0.2"
 subst = { version = "0.3", features = ["yaml"] }
 testcontainers-modules = { version = "0.11.4", features = ["postgres"] }
 thiserror = "2"
+tiff = "0.9.1"
 tile-grid = "0.6"
 tilejson = "0.4"
 tokio = { version = "1", features = ["macros"] }

docs/src/SUMMARY.md

@@ -20,6 +20,7 @@
   - [PostgreSQL Table Sources](sources-pg-tables.md)
   - [PostgreSQL Function Sources](sources-pg-functions.md)
   - [MBTiles and PMTiles File Sources](sources-files.md)
+  - [Cloud Optimized GeoTIFF File Sources](sources-cog-files.md)
   - [Composite Sources](sources-composite.md)
   - [Sprite Sources](sources-sprites.md)
   - [Font Sources](sources-fonts.md)

docs/src/config-file.md

@@ -199,6 +199,18 @@ mbtiles:
     # named source matching source name to a single file
     mb-src1: /path/to/mbtiles1.mbtiles
 
+# Cloud Optimized GeoTIFF File Sources
+cog:
+  paths:
+    # scan this whole dir, matching all *.tif files
+    - /dir-path
+    # specific TIFF file will be published as a cog source
+    - /path/to/cogfile.tif
+  sources:
+    # named source matching source name to a single file
+     cog-src1: /path/to/cog1.tif
+     cog-src2: /path/to/cog2.tif
+
 # Sprite configuration
 sprites:
   paths:

docs/src/sources-cog-files.md

@@ -0,0 +1,104 @@
+# Cloud Optimized GeoTIFF File Sources
+
+Martin can also serve raster sources like local [COG(Cloud Optimized GeoTIFF)](https://cogeo.org/) files. For cog on remote like S3 and other improvements, you could track them on [issue 875](https://github.com/maplibre/martin/issues/875), we are working on and welcome any assistance.
+
+## Supported colortype and bits per sample
+
+| colory type | bits per sample | supported | status     |
+| ----------- | --------------- | --------- | ---------- |
+| rgb/rgba    | 8               | ✅         |            |
+| rgb/rgba    | 16/32...        | 🛠️         | working on |
+| gray        | 8/16/32...      | 🛠️         | working on |
+
+## Supported compression
+
+* None
+* LZW
+* Deflate
+* PackBits
+
+## Run Martin with CLI to serve cog files
+
+```bash
+# Configured with a directory containing TIFF files.
+martin /with/tiff/dir1 /with/tiff/dir2
+# Configured with dedicated TIFF file.
+martin /path/to/target1.tif /path/to/target1.tif
+# Configured with a combination of directories and dedicated TIFF files.
+martin /with/tiff/files /path/to/target.tif
+```
+
+## Run Martin with configuration file
+
+```yml
+keep_alive: 75
+
+# The socket address to bind [default: 0.0.0.0:3000]
+listen_addresses: '0.0.0.0:3000'
+
+# Number of web server workers
+worker_processes: 8
+
+# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable]
+cache_size_mb: 8
+
+# Database configuration. This can also be a list of PG configs.
+
+cog:
+  paths:
+    # scan this whole dir, matching all *.tif files
+    - /dir-path
+    # specific TIFF file will be published as a cog source
+    - /path/to/target1.tif
+    - /path/to/target2.tif
+  sources:
+    # named source matching source name to a single file
+     cog-src1: /path/to/cog1.tif
+     cog-src2: /path/to/cog2.tif
+```
+
+## About COG
+
+[COG](https://cogeo.org/) is just Cloud Optimized GeoTIFF file.
+
+TIFF is an image file format. TIFF tags are something like key-value pairs inside to describe the metadata about a TIFF file, ike `ImageWidth`, `ImageLength`, etc.
+
+GeoTIFF is a valid TIFF file with a set of TIFF tags to describe the 'Cartographic' information associated with it.
+
+COG is a valid GeoTIFF file with some requirements for efficient reading. That is, all COG files are valid GeoTIFF files, but not all GeoTIFF files are valid COG files. For quick access to tiles in TIFF files, Martin relies on the requirements/recommendations(like the [requirement about Reduced-Resolution Subfiles](https://docs.ogc.org/is/21-026/21-026.html#_requirement_reduced_resolution_subfiles) and [the content dividing strategy](https://docs.ogc.org/is/21-026/21-026.html#_tiles)) so we use the term `COG` over `GeoTIFF` in our documentation and configuration files.
+
+You may want to visit these specs:
+
+* [TIFF 6.0](https://www.itu.int/itudoc/itu-t/com16/tiff-fx/docs/tiff6.pdf)
+* [GeoTIFF](https://docs.ogc.org/is/19-008r4/19-008r4.html)
+* [Cloud Optimized GeoTIFF](https://docs.ogc.org/is/21-026/21-026.html)
+
+### COG generation with GDAL
+
+You could generate cog with `gdal_translate` or `gdalwarp`. See more details in [gdal doc](https://gdal.org/en/latest/drivers/raster/cog.html).
+
+```bash
+# gdal-bin installation
+# sudo apt update
+# sudo apt install gdal-bin
+
+# gdalwarp
+gdalwarp src1.tif src2.tif out.tif -of COG
+
+# or gdal_translate
+gdal_translate input.tif output_cog.tif -of COG
+```
+
+### The mapping from ZXY to tiff chunk
+
+* A single TIFF file could contains many sub-file about same spatial area, each has different resolution
+* A sub file is organized with many tiles
+
+So basically there's a mapping from zxy to tile of sub-file of TIFF.
+
+| zxy        | mapping to                  |
+| ---------- | --------------------------- |
+| Zoom level | which sub-file in TIFF file |
+| X and Y    | which tile in subfile       |
+
+Clients could read only the header part of COG to figure out the mapping from zxy to the chunk number and the subfile number. Martin get tile to frontend by this mapping.

martin/Cargo.toml

@@ -59,12 +59,13 @@ name = "bench"
 harness = false
 
 [features]
-default = ["webui", "fonts", "lambda", "mbtiles", "pmtiles", "postgres", "sprites"]
+default = ["webui", "fonts", "lambda", "mbtiles", "pmtiles", "cog", "postgres", "sprites"]
 webui = ["dep:actix-web-static-files", "dep:static-files"]
 fonts = ["dep:bit-set", "dep:pbf_font_tools"]
 lambda = ["dep:lambda-web"]
 mbtiles = ["dep:mbtiles"]
 pmtiles = ["dep:pmtiles"]
+cog = ["dep:tiff", "dep:png"]
 postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"]
 sprites = ["dep:spreet", "tokio/fs"]
 bless-tests = []
@@ -93,6 +94,7 @@ moka.workspace = true
 num_cpus.workspace = true
 pbf_font_tools = { workspace = true, optional = true }
 pmtiles = { workspace = true, optional = true }
+png= { workspace = true, optional=true }
 postgis = { workspace = true, optional = true }
 postgres = { workspace = true, optional = true }
 postgres-protocol = { workspace = true, optional = true }
@@ -101,14 +103,15 @@ rustls-native-certs.workspace = true
 rustls-pemfile.workspace = true
 rustls.workspace = true
 semver = { workspace = true, optional = true }
-serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 serde_yaml.workspace = true
+serde.workspace = true
 spreet = { workspace = true, optional = true }
 static-files = { workspace = true, optional = true }
 subst.workspace = true
 thiserror.workspace = true
+tiff= { workspace = true, optional=true }
 tilejson.workspace = true
 tokio = { workspace = true, features = ["io-std"] }
 tokio-postgres-rustls = { workspace = true, optional = true }

martin/src/args/root.rs

@@ -107,6 +107,11 @@ impl Args {
             config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false);
         }
 
+        #[cfg(feature = "cog")]
+        if !cli_strings.is_empty() {
+            config.cog = parse_file_args(&mut cli_strings, "tif", false);
+        }
+
         #[cfg(feature = "sprites")]
         if !self.extras.sprite.is_empty() {
             config.sprites = FileConfigEnum::new(self.extras.sprite);

martin/src/cog/errors.rs

@@ -0,0 +1,51 @@
+use std::path::PathBuf;
+
+use png::EncodingError;
+use tiff::TiffError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum CogError {
+    #[error("Couldn't decode {1} as tiff file: {0}")]
+    InvalidTiffFile(TiffError, PathBuf),
+
+    #[error("Requested zoom level {0} from file {1} is out of range. Possible zoom levels are {2} to {3}")]
+    ZoomOutOfRange(u8, PathBuf, u8, u8),
+
+    #[error("Couldn't find any image in the tiff file: {0}")]
+    NoImagesFound(PathBuf),
+
+    #[error("Couldn't seek to ifd number {1} (0 based indexing) in tiff file {2}: {0}")]
+    IfdSeekFailed(TiffError, usize, PathBuf),
+
+    #[error("Too many images in the tiff file: {0}")]
+    TooManyImages(PathBuf),
+
+    #[error("Couldn't find tags {1:?} at ifd {2} of tiff file {3}: {0}")]
+    TagsNotFound(TiffError, Vec<u16>, usize, PathBuf),
+
+    #[error(
+        "Unsupported planar configuration {2} at IFD {1} in TIFF file {0}. Only planar configuration 1 is supported."
+    )]
+    PlanarConfigurationNotSupported(PathBuf, usize, u16),
+
+    #[error("Failed to read {1}th chunk(0 based index) at ifd {2} from tiff file {3}: {0}")]
+    ReadChunkFailed(TiffError, u32, usize, PathBuf),
+
+    #[error("Failed to write header of png file at {0}: {1}")]
+    WritePngHeaderFailed(PathBuf, EncodingError),
+
+    #[error("Failed to write pixel bytes to png file at {0}: {1}")]
+    WriteToPngFailed(PathBuf, EncodingError),
+
+    #[error("The color type {0:?} and its bit depth of the tiff file {1} is not supported yet")]
+    NotSupportedColorTypeAndBitDepth(tiff::ColorType, PathBuf),
+
+    #[error("Couldn't parse the {0} value in gdal metadata(tiff tag 42112) from {1}")]
+    ParseSTATISTICSValueFailed(String, PathBuf),
+
+    #[error("The gdal metadata(tiff tag 42112) from {1} is not valid: {0}")]
+    InvalidGdalMetaData(String, PathBuf),
+
+    #[error("Striped tiff file is not supported, the tiff file is {0}")]
+    NotSupportedChunkType(PathBuf),
+}