Add martin-cp tool to bulk-copy tiles from any Martin-supported sou…

…rces (#1011)

This adds a new utility called `martin-cp` that allows copying any
number of tiles from a single source (or a composite source) to an
mbtiles file.

`martin-cp` is a tool for generating tiles in bulk, and save retrieved
tiles into a new or an existing MBTiles file. It can be used to generate
tiles for a large area or multiple areas. If multiple areas overlap, it
will generate tiles only once. `martin-cp` supports the same
configuration file and CLI arguments as Martin server, so it can support
all sources and even combining sources.

.github/workflows/ci.yml

@@ -148,8 +148,9 @@ jobs:
             cross build --release --target $target --package martin
 
             mkdir -p target_releases/$target
-            mv target/$target/release/mbtiles target_releases/$target
             mv target/$target/release/martin target_releases/$target
+            mv target/$target/release/martin-cp target_releases/$target
+            mv target/$target/release/mbtiles target_releases/$target
           done
 
       - name: Save build artifacts to build-${{ matrix.target }}
@@ -179,6 +180,7 @@ jobs:
           TAG=${{ github.repository }}:linux-arm64
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
+          export MARTIN_CP_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/martin-cp $TAG"
           export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
           tests/test.sh
         env:
@@ -199,6 +201,7 @@ jobs:
           TAG=${{ github.repository }}:linux-amd64
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
+          export MARTIN_CP_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/martin-cp $TAG"
           export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
           tests/test.sh
         env:
@@ -275,8 +278,9 @@ jobs:
           cargo build --release --target ${{ matrix.target }} --package mbtiles
           cargo build --release --target ${{ matrix.target }} --package martin
           mkdir -p target_releases
-          mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/
           mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases/
+          mv target/${{ matrix.target }}/release/martin-cp${{ matrix.ext }} target_releases/
+          mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/
       - name: Save build artifacts to build-${{ matrix.target }}
         uses: actions/upload-artifact@v3
         with:
@@ -328,9 +332,10 @@ jobs:
         run: |
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN=target/martin${{ matrix.ext }}
+          export MARTIN_CP_BIN=target/martin-cp${{ matrix.ext }}
           export MBTILES_BIN=target/mbtiles${{ matrix.ext }}
           if [[ "${{ runner.os }}" != "Windows" ]]; then
-            chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
+            chmod +x "$MARTIN_BIN" "$MARTIN_CP_BIN" "$MBTILES_BIN"
           fi
           tests/test.sh
         env:
@@ -350,6 +355,7 @@ jobs:
           sudo dpkg -i target/debian-x86_64.deb
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN=/usr/bin/martin${{ matrix.ext }}
+          export MARTIN_CP_BIN=/usr/bin/martin-cp${{ matrix.ext }}
           export MBTILES_BIN=/usr/bin/mbtiles${{ matrix.ext }}
           tests/test.sh
         env:
@@ -444,8 +450,9 @@ jobs:
           fi
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN=target_releases/martin
+          export MARTIN_CP_BIN=target_releases/martin-cp
           export MBTILES_BIN=target_releases/mbtiles
-          chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
+          chmod +x "$MARTIN_BIN" "$MARTIN_CP_BIN" "$MBTILES_BIN"
           tests/test.sh
           rm -rf target_releases
         env:
@@ -463,6 +470,7 @@ jobs:
           fi
           export MARTIN_BUILD_ALL=-
           export MARTIN_BIN=/usr/bin/martin
+          export MARTIN_CP_BIN=/usr/bin/martin-cp
           export MBTILES_BIN=/usr/bin/mbtiles
           tests/test.sh
           sudo dpkg -P martin

Cargo.toml

@@ -17,7 +17,7 @@ unused_qualifications = "warn"
 # unsafe_code = "forbid"
 
 [workspace.lints.clippy]
-pedantic = "warn"
+pedantic = { level = "warn", priority = -1 }
 derive_partial_eq_without_eq = "allow"
 implicit_hasher = "allow"
 missing_errors_doc = "allow"
@@ -49,10 +49,10 @@ itertools = "0.12"
 json-patch = "1.2"
 log = "0.4"
 martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" }
-mbtiles = { path = "./mbtiles", version = "0.7.0", default-features = false }
+mbtiles = { path = "./mbtiles", version = "0.7.0" }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.0", features = ["freetype"] }
-pmtiles = { version = "0.4", features = ["mmap-async-tokio", "tilejson"] }
+pmtiles = { version = "0.5", features = ["mmap-async-tokio", "tilejson"] }
 postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-protocol = "0.6"
@@ -73,7 +73,7 @@ sqlite-hashes = { version = "0.5", default-features = false, features = ["md5",
 sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio"] }
 subst = { version = "0.3", features = ["yaml"] }
 thiserror = "1"
-tilejson = "0.3"
+tilejson = "0.4"
 tokio = { version = "1", features = ["macros"] }
 tokio-postgres-rustls = "0.10"
 
@@ -84,7 +84,9 @@ sqlx-macros.opt-level = 3
 insta.opt-level = 3
 similar.opt-level = 3
 
-#[patch.crates-io]
+[patch.crates-io]
 #pmtiles = { path = "../pmtiles-rs" }
-#sqlite-hashes = { path = "/home/nyurik/dev/rust/sqlite-hashes" }
+#sqlite-hashes = { path = "../sqlite-hashes" }
 #tilejson = { path = "../tilejson" }
+#pmtiles = { git = "https://github.com/nyurik/pmtiles-rs", branch = "override" }
+#tilejson = { git = "https://github.com/nyurik/tilejson", branch = "btreemap" }

README.md

@@ -11,6 +11,8 @@
 
 Martin is a tile server able to generate and serve [vector tiles](https://github.com/mapbox/vector-tile-spec) on the fly from large [PostGIS](https://github.com/postgis/postgis) databases, [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new), and [MBTile](https://github.com/mapbox/mbtiles-spec) files, allowing multiple tile sources to be dynamically combined into one. Martin optimizes for speed and heavy traffic, and is written in [Rust](https://github.com/rust-lang/rust).
 
+Additionally, there are [several tools](https://maplibre.org/martin/50-tools.html) for generating tiles in bulk from any Martin-supported sources (similar to `tilelive-copy`), copying tiles between MBTiles files, creating deltas (patches) and applying them, and validating MBTiles files.
+
 See [Martin book](https://maplibre.org/martin/) for complete documentation.
 
 ## Installation

docs/src/50-tools.md

@@ -2,6 +2,9 @@
 
 Martin project contains additional tooling to help manage the data servable with Martin tile server.
 
+## `martin-cp`
+`martin-cp` is a tool for generating tiles in bulk, and save retrieved tiles into a new or an existing MBTiles file. It can be used to generate tiles for a large area or multiple areas. If multiple areas overlap, it will generate tiles only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources.
+
 ## `mbtiles`
 `mbtiles` is a small utility to interact with the `*.mbtiles` files from the command line. It allows users to examine, copy, validate, compare, and apply diffs between them.
 

docs/src/501-martin-cp.md

@@ -0,0 +1,17 @@
+# Generating Tiles in Bulk
+
+`martin-cp` is a tool for generating tiles in bulk, and save retrieved tiles into a new or an existing MBTiles file. It can be used to generate tiles for a large area or multiple areas. If multiple areas overlap, it will generate tiles only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources.
+
+## Usage
+
+This copies tiles from a PostGIS table `my_table` into an MBTiles file `tileset.mbtiles` using [normalized](54-mbtiles-schema.md) schema, with zoom levels from 0 to 10, and bounds of the whole world. 
+
+```shell
+martin-cp  --output-file tileset.mbtiles \
+           --dst-type normalized         \
+           "--bbox=-180,-90,180,90"      \
+           --min-zoom 0                  \
+           --max-zoom 10                 \
+           --source my_table             \
+           postgresql://postgres@localhost:5432/db
+```

docs/src/SUMMARY.md

@@ -22,6 +22,7 @@
   - [Using with Mapbox](44-using-with-mapbox.md)
   - [Recipes](45-recipes.md)
 - [Tools](50-tools.md)
+  - [martin-cp bulk tile generation](501-martin-cp.md)
   - [MBTiles Info and Metadata](51-mbtiles-meta.md)
   - [MBTiles Copying / Diffing](52-mbtiles-copy.md)
   - [MBTiles Validation](53-mbtiles-validation.md)

justfile

@@ -19,6 +19,10 @@ export CARGO_TERM_COLOR := "always"
 run *ARGS:
     cargo run -p martin -- {{ ARGS }}
 
+# Start Martin server
+cp *ARGS:
+    cargo run --bin martin-cp -- {{ ARGS }}
+
 # Run mbtiles command
 mbtiles *ARGS:
     cargo run -p mbtiles -- {{ ARGS }}
@@ -167,6 +171,10 @@ bless-insta-mbtiles *ARGS: (cargo-install "cargo-insta")
 bless-insta-martin *ARGS: (cargo-install "cargo-insta")
     cargo insta test --accept --unreferenced=auto -p martin {{ ARGS }}
 
+# Run integration tests and save its output as the new expected output
+bless-insta-cp *ARGS: (cargo-install "cargo-insta")
+    cargo insta test --accept --bin martin-cp {{ ARGS }}
+
 # Build and open mdbook documentation
 book: (cargo-install "mdbook")
     mdbook serve docs --open --port 8321

martin/Cargo.toml

@@ -28,6 +28,7 @@ maintainer-scripts = "../debian"
 depends = "$auto"
 assets = [
     ["target/release/martin", "/usr/bin/martin", "755"],
+    ["target/release/martin-cp", "/usr/bin/martin-cp", "755"],
     ["target/release/mbtiles", "/usr/bin/mbtiles", "755"],
     ["../README.md", "/usr/share/doc/martin/README.md", "644"],
     ["../debian/config.yaml", "/usr/share/doc/martin/config.yaml", "644"],
@@ -49,6 +50,10 @@ path = "src/lib.rs"
 name = "martin"
 path = "src/bin/martin.rs"
 
+[[bin]]
+name = "martin-cp"
+path = "src/bin/martin-cp.rs"
+
 [features]
 default = []
 bless-tests = []