PMTiles cache, refactor file configs, modularize

* Implement PMTiles directory cache shared between all pmtiles, with configurable max cache size (in MB), or 0 to disable.
* PMTiles now share web client instance, which optimizes connection reuse in case multiple pmtiles reside on the same host
* Major refactoring to allow modular reuse, enabling the following build features:
    * **postgres** - enable PostgreSQL/PostGIS tile sources
    * **pmtiles** - enable PMTile tile sources
    * **mbtiles** - enable MBTile tile sources
    * **fonts** - enable font sources
    * **sprites** - enable sprite sources
* Use justfile in the CI

.github/workflows/ci.yml

@@ -49,24 +49,18 @@ jobs:
           postgis/postgis:16-3.4
           -c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
     steps:
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: just
       - name: Checkout sources
         uses: actions/checkout@v4
-      - name: Rust Versions
-        run: rustc --version && cargo --version
       - uses: Swatinem/rust-cache@v2
         if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
-      - run: cargo fmt --all -- --check
-      - run: cargo clippy --package martin-tile-utils --tests --lib --bins --benches -- -D warnings
-      - run: cargo clippy --package mbtiles --tests --lib --bins --benches --no-default-features -- -D warnings
-      - run: cargo clippy --package mbtiles --tests --lib --bins --benches -- -D warnings
-      - run: cargo clippy --package martin --tests --lib --bins --benches -- -D warnings
-      - run: RUSTFLAGS='-D warnings' cargo check --package martin --tests --lib --bins --benches --no-default-features
-      - run: RUSTFLAGS='-D warnings' cargo check --package martin --tests --lib --bins --benches --no-default-features --features fonts
-      - run: RUSTFLAGS='-D warnings' cargo check --package martin --tests --lib --bins --benches --no-default-features --features sprites
-      - run: cargo clippy --package martin --features bless-tests -- -D warnings
-      - run: cargo doc --no-deps --workspace
-        env:
-          RUSTDOCFLAGS: '-D warnings'
+      - run: just env-info
+      - run: just fmt
+      - run: just clippy
+      - run: just check
+      - run: just check-doc
       - name: Init database
         run: tests/fixtures/initdb.sh
         env:

Cargo.toml

@@ -48,8 +48,8 @@ insta = "1"
 itertools = "0.12"
 json-patch = "1.2"
 log = "0.4"
-martin-tile-utils = { path = "./martin-tile-utils", version = "0.3.0" }
-mbtiles = { path = "./mbtiles", version = "0.8.0" }
+martin-tile-utils = { path = "./martin-tile-utils", version = "0.4.0" }
+mbtiles = { path = "./mbtiles", version = "0.9.0" }
 moka = { version = "0.12", features = ["future"] }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.0", features = ["freetype"] }

README.md

@@ -106,6 +106,14 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/font/{font1},…,{fontN}/{start}-{end}` | Composite Font source                         |
 | `/health`                               | Martin server health check: returns 200 `OK`  |
 
+## Re-use Martin as a library
+Martin can be used as a standalone server, or as a library in your own Rust application. When used as a library, you can use the following features:
+* **postgres** - enable PostgreSQL/PostGIS tile sources
+* **pmtiles** - enable PMTile tile sources
+* **mbtiles** - enable MBTile tile sources
+* **fonts** - enable font sources
+* **sprites** - enable sprite sources
+
 ## Documentation
 
 See [Martin book](https://maplibre.org/martin/) for complete documentation.

debian/config.yaml

@@ -17,7 +17,7 @@ worker_processes: 8
 #   auto_bounds: skip
 
 # pmtiles:
-#   dir_cache_size: 100
+#   dir_cache_size_mb: 100
 #   paths:
 #     - /dir-path
 #     - /path/to/pmtiles.pmtiles
@@ -33,6 +33,9 @@ worker_processes: 8
 #   sources:
 #     mb-src1: /path/to/mbtiles1.mbtiles
 
+# sprites:
+#   - /path/to/sprites_dir
+
 # fonts:
 #   - /path/to/font/file.ttf
 #   - /path/to/font_dir

docs/src/config-file.md

@@ -155,6 +155,8 @@ postgres:
 
 # Publish PMTiles files from local disk or proxy to a web server
 pmtiles:
+  # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] 
+  dir_cache_size_mb: 100
   paths:
     # scan this whole dir, matching all *.pmtiles files
     - /dir-path

justfile

@@ -273,21 +273,31 @@ fmt2:
 
 # Run cargo check
 check:
-    cargo check --workspace --all-targets --bins --tests --lib --benches
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin-tile-utils
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles --no-default-features
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features fonts
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features mbtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features pmtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features postgres
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features sprites
+
+check-doc:
+    RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace
 
 # Run cargo clippy
 clippy:
     cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings
-    RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace
 
 # Validate markdown URLs with markdown-link-check
 clippy-md:
     docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c \
       'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")" | tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}'
 
 # These steps automatically run before git push via a git hook
-[private]
-git-pre-push: env-info restart lint test
+git-pre-push: env-info restart fmt clippy check check-doc test
 
 # Get environment info
 [private]
@@ -296,6 +306,7 @@ env-info:
     {{ just_executable() }} --version
     rustc --version
     cargo --version
+    rustup --version
 
 # Update sqlite database schema.
 prepare-sqlite: install-sqlx

martin-tile-utils/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin-tile-utils"
-version = "0.3.1"
+version = "0.4.0"
 authors = ["Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server."
 keywords = ["maps", "tiles", "mvt", "tileserver"]

martin/Cargo.toml

@@ -3,7 +3,7 @@ lints.workspace = true
 [package]
 name = "martin"
 # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin
-version = "0.11.6"
+version = "0.12.0"
 authors = ["Stepan Kuzmin <[email protected]>", "Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"
 keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]
@@ -59,10 +59,12 @@ name = "bench"
 harness = false
 
 [features]
-default = []
-#default = ["sprites", "fonts"]
-sprites = []
-fonts = []
+default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"]
+fonts = ["dep:bit-set","dep:pbf_font_tools"]
+mbtiles = []
+pmtiles = ["dep:moka"]
+postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"]
+sprites = ["dep:spreet"]
 bless-tests = []
 
 [dependencies]
@@ -71,41 +73,41 @@ actix-http.workspace = true
 actix-rt.workspace = true
 actix-web.workspace = true
 async-trait.workspace = true
-bit-set.workspace = true
+bit-set = { workspace = true, optional = true }
 brotli.workspace = true
 clap.workspace = true
-deadpool-postgres.workspace = true
+deadpool-postgres = { workspace = true, optional = true }
 env_logger.workspace = true
 flate2.workspace = true
 futures.workspace = true
 itertools.workspace = true
-json-patch.workspace = true
+json-patch = { workspace = true, optional = true }
 log.workspace = true
 martin-tile-utils.workspace = true
 mbtiles.workspace = true
-moka.workspace = true
+moka = { workspace = true, optional = true }
 num_cpus.workspace = true
-pbf_font_tools.workspace = true
+pbf_font_tools = { workspace = true, optional = true }
 pmtiles.workspace = true
-postgis.workspace = true
-postgres-protocol.workspace = true
-postgres.workspace = true
+postgis = { workspace = true, optional = true }
+postgres-protocol = { workspace = true, optional = true }
+postgres = { workspace = true, optional = true }
 regex.workspace = true
 reqwest.workspace = true
 rustls-native-certs.workspace = true
 rustls-pemfile.workspace = true
 rustls.workspace = true
-semver.workspace = true
+semver = { workspace = true, optional = true }
 serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 serde_yaml.workspace = true
-spreet.workspace = true
+spreet = { workspace = true, optional = true }
 subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
 tokio = { workspace = true, features = ["io-std"] }
-tokio-postgres-rustls.workspace = true
+tokio-postgres-rustls = { workspace = true, optional = true }
 url.workspace = true
 
 [dev-dependencies]

martin/src/args/mod.rs

@@ -4,7 +4,9 @@ pub use connections::{Arguments, State};
 mod environment;
 pub use environment::{Env, OsEnv};
 
+#[cfg(feature = "postgres")]
 mod pg;
+#[cfg(feature = "postgres")]
 pub use pg::{BoundsCalcType, PgArgs, DEFAULT_BOUNDS_TIMEOUT};
 
 mod root;