Add binary diff for mbtiles (#1358)

* `mbtiles diff` now has an additional `--patch-type` param with
`whole`, `bin-diff-raw` and `bin-diff-gz` values:
* `whole` stores different tiles as before - as whole tiles in the
`tiles` table
* `bin-diff-raw` computes binary difference between tiles, and stores
them as brotli-encoded value in a `bsdiffraw` table, together with a
`xxh3_64` hash of the tile as it will be stored after patching
* `bin-diff-gz` same as `bin-diff-raw`, but assumes the tiles are
gzip-compressed, so it uncompresses them before comparing. The `xxh3_64`
stores the hash of the uncompressed tile. The data will be stored in the
`bsdiffrawgz` table (identical structure with above)

* `mbtiles copy --apply-patch` will automatically detect if
`bsdiffrawgz` or `bsdiffraw` tables exist, and will use binary patching.
* `mbtiles apply-patch` does not support binary patching yet
* `mbtiles copy --diff-with-file ... --patch-type ...` is an alias to
`mbtiles diff --patch-type ...`

.gitignore

@@ -10,5 +10,6 @@ test_log*
 pg_data/
 config.yml
 tests/output/
+tests/mbtiles_temp_files/
 tmp/
 .aws-sam/

Cargo.toml

@@ -6,7 +6,7 @@ members = ["martin", "martin-tile-utils", "mbtiles"]
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/maplibre/martin"
-rust-version = "1.74"
+rust-version = "1.76"
 readme = "README.md"
 homepage = "https://martin.maplibre.org/"
 
@@ -42,6 +42,7 @@ deadpool-postgres = "0.12"
 enum-display = "0.1"
 env_logger = "0.11"
 flate2 = "1"
+flume = "0.11"
 futures = "0.3"
 indoc = "2"
 insta = "1"
@@ -50,8 +51,9 @@ json-patch = "2.0"
 lambda-web = { version = "0.2.1", features = ["actix4"] }
 libsqlite3-sys = { version = ">=0.27", features = ["bundled"] }
 log = "0.4"
-martin-tile-utils = { path = "./martin-tile-utils", version = "0.4.0" }
-mbtiles = { path = "./mbtiles", version = "0.9.0" }
+martin-tile-utils = { path = "./martin-tile-utils", version = "0.5.0" }
+mbtiles = { path = "./mbtiles", version = "0.10.0" }
+md5 = "0.7.0"
 moka = { version = "0.12", features = ["future"] }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.1", features = ["freetype"] }
@@ -75,6 +77,7 @@ serde_with = "3"
 serde_yaml = "0.9"
 size_format = "1.0.2"
 spreet = { version = "0.11", default-features = false }
+sqlite-compressions = { version = "0.2.12", default-features = false, features = ["bsdiffraw", "gzip"] }
 sqlite-hashes = { version = "0.7.3", default-features = false, features = ["md5", "aggregate", "hex"] }
 sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio"] }
 subst = { version = "0.3", features = ["yaml"] }
@@ -84,6 +87,7 @@ tilejson = "0.4"
 tokio = { version = "1", features = ["macros"] }
 tokio-postgres-rustls = "0.12"
 url = "2.5"
+xxhash-rust = { version = "0.8", features = ["xxh3"] }
 
 [profile.dev.package]
 # See https://github.com/launchbadge/sqlx#compile-time-verification

justfile

@@ -188,8 +188,7 @@ bless: restart clean-test bless-insta-martin bless-insta-mbtiles bless-tests ble
 bless-int:
     rm -rf tests/temp
     tests/test.sh
-    rm -rf tests/expected
-    mv tests/output tests/expected
+    rm -rf tests/expected && mv tests/output tests/expected
 
 # Run test with bless-tests feature
 bless-tests:

martin-tile-utils/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin-tile-utils"
-version = "0.4.1"
+version = "0.5.0"
 authors = ["Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Utilities to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server."
 keywords = ["maps", "tiles", "mvt", "tileserver"]
@@ -17,6 +17,8 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
+brotli.workspace = true
+flate2.workspace = true
 
 [dev-dependencies]
 approx.workspace = true

martin-tile-utils/src/decoders.rs

@@ -0,0 +1,30 @@
+use std::io::{Read as _, Write as _};
+
+use flate2::read::GzDecoder;
+use flate2::write::GzEncoder;
+
+pub fn decode_gzip(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+    let mut decoder = GzDecoder::new(data);
+    let mut decompressed = Vec::new();
+    decoder.read_to_end(&mut decompressed)?;
+    Ok(decompressed)
+}
+
+pub fn encode_gzip(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+    let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default());
+    encoder.write_all(data)?;
+    encoder.finish()
+}
+
+pub fn decode_brotli(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+    let mut decoder = brotli::Decompressor::new(data, 4096);
+    let mut decompressed = Vec::new();
+    decoder.read_to_end(&mut decompressed)?;
+    Ok(decompressed)
+}
+
+pub fn encode_brotli(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+    let mut encoder = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22);
+    encoder.write_all(data)?;
+    Ok(encoder.into_inner())
+}

martin-tile-utils/src/lib.rs

@@ -4,13 +4,33 @@
 // project originally written by Kaveh Karimi and licensed under MIT/Apache-2.0
 
 use std::f64::consts::PI;
-use std::fmt::Display;
+use std::fmt::{Display, Formatter, Result};
 
 pub const EARTH_CIRCUMFERENCE: f64 = 40_075_016.685_578_5;
 pub const EARTH_RADIUS: f64 = EARTH_CIRCUMFERENCE / 2.0 / PI;
 
 pub const MAX_ZOOM: u8 = 30;
 
+mod decoders;
+pub use decoders::*;
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct TileCoord {
+    pub z: u8,
+    pub x: u32,
+    pub y: u32,
+}
+
+impl Display for TileCoord {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        if f.alternate() {
+            write!(f, "{}/{}/{}", self.z, self.x, self.y)
+        } else {
+            write!(f, "{},{},{}", self.z, self.x, self.y)
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Format {
     Gif,
@@ -74,7 +94,7 @@ impl Format {
 }
 
 impl Display for Format {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
         f.write_str(match *self {
             Self::Gif => "gif",
             Self::Jpeg => "jpeg",
@@ -189,7 +209,7 @@ impl From<Format> for TileInfo {
 }
 
 impl Display for TileInfo {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
         write!(f, "{}", self.format.content_type())?;
         if let Some(encoding) = self.encoding.content_encoding() {
             write!(f, "; encoding={encoding}")?;

martin/Cargo.toml

@@ -3,7 +3,7 @@ lints.workspace = true
 [package]
 name = "martin"
 # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin
-version = "0.13.0"
+version = "0.14.0"
 authors = ["Stepan Kuzmin <[email protected]>", "Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"
 keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]
@@ -81,7 +81,6 @@ clap.workspace = true
 deadpool-postgres = { workspace = true, optional = true }
 enum-display.workspace = true
 env_logger.workspace = true
-flate2.workspace = true
 futures.workspace = true
 itertools.workspace = true
 json-patch = { workspace = true, optional = true }

martin/benches/bench.rs

@@ -2,10 +2,8 @@ use async_trait::async_trait;
 use criterion::async_executor::FuturesExecutor;
 use criterion::{criterion_group, criterion_main, Criterion};
 use martin::srv::DynTileSource;
-use martin::{
-    CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery,
-};
-use martin_tile_utils::{Encoding, Format, TileInfo};
+use martin::{CatalogSourceEntry, MartinResult, Source, TileData, TileSources, UrlQuery};
+use martin_tile_utils::{Encoding, Format, TileCoord, TileInfo};
 use pprof::criterion::{Output, PProfProfiler};
 use tilejson::{tilejson, TileJSON};
 

martin/src/bin/martin-cp.rs

@@ -14,10 +14,10 @@ use log::{debug, error, info, log_enabled};
 use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs};
 use martin::srv::{merge_tilejson, DynTileSource};
 use martin::{
-    append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord,
-    TileData, TileRect,
+    append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileData,
+    TileRect,
 };
-use martin_tile_utils::{bbox_to_xyz, TileInfo};
+use martin_tile_utils::{bbox_to_xyz, TileCoord, TileInfo};
 use mbtiles::sqlx::SqliteConnection;
 use mbtiles::UpdateZoomType::GrowOnly;
 use mbtiles::{

martin/src/lib.rs

@@ -9,8 +9,8 @@ pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQue
 
 mod utils;
 pub use utils::{
-    append_rect, decode_brotli, decode_gzip, IdResolver, MartinError, MartinResult, OptBoolObj,
-    OptOneMany, TileCoord, TileRect, NO_MAIN_CACHE,
+    append_rect, IdResolver, MartinError, MartinResult, OptBoolObj, OptOneMany, TileRect,
+    NO_MAIN_CACHE,
 };
 
 pub mod args;

martin/src/mbtiles/mod.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use log::trace;
-use martin_tile_utils::TileInfo;
+use martin_tile_utils::{TileCoord, TileInfo};
 use mbtiles::MbtilesPool;
 use serde::{Deserialize, Serialize};
 use tilejson::TileJSON;
@@ -15,7 +15,7 @@ use crate::config::UnrecognizedValues;
 use crate::file_config::FileError::{AcquireConnError, InvalidMetadata, IoError};
 use crate::file_config::{ConfigExtras, FileResult, SourceConfigExtras};
 use crate::source::{TileData, UrlQuery};
-use crate::{MartinResult, Source, TileCoord};
+use crate::{MartinResult, Source};
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct MbtConfig {