From b71c846abc620e69adc2d82bc3f39d499608b34f Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 26 Jun 2024 16:37:20 -0400 Subject: [PATCH] fix for mbtiles apply-patch with raw bindiff (#1384) When applying raw (no gzip) patches, `mbtiles` was trying to un-gzip them first. Now handles it properly. Also adds a number of tests to catch these cases. --- Cargo.lock | 2 +- Cargo.toml | 2 +- mbtiles/Cargo.toml | 2 +- mbtiles/src/bin/mbtiles.rs | 17 ++- mbtiles/src/bindiff.rs | 74 ++++++++++--- mbtiles/src/copier.rs | 95 +++++++--------- mbtiles/src/errors.rs | 2 +- mbtiles/src/lib.rs | 3 +- mbtiles/src/mbtiles.rs | 5 +- mbtiles/src/patcher.rs | 3 +- mbtiles/src/queries.rs | 10 +- mbtiles/tests/copy.rs | 104 +++++++++++++----- .../snapshots/copy__databases@flat__bdz.snap | 67 +++++++++++ .../snapshots/copy__databases@flat__v1z.snap | 50 +++++++++ .../snapshots/copy__databases@flat__v2-2.snap | 49 +++++++++ .../snapshots/copy__databases@hash__bdz.snap | 75 +++++++++++++ .../snapshots/copy__databases@hash__v1z.snap | 58 ++++++++++ .../snapshots/copy__databases@hash__v2-2.snap | 57 ++++++++++ .../snapshots/copy__databases@norm__v1z.snap | 97 ++++++++++++++++ .../snapshots/copy__databases@norm__v2-2.snap | 97 ++++++++++++++++ tests/expected/mbtiles/copy_bindiff.txt | 4 +- tests/expected/mbtiles/copy_bindiff2.txt | 4 +- tests/expected/mbtiles/copy_diff.txt | 2 +- .../mbtiles/world_cities_diff.mbtiles | Bin 4096 -> 4096 bytes .../mbtiles/world_cities_modified.mbtiles | Bin 49152 -> 49152 bytes 25 files changed, 754 insertions(+), 125 deletions(-) create mode 100644 mbtiles/tests/snapshots/copy__databases@flat__bdz.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@flat__v1z.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@flat__v2-2.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@hash__bdz.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@hash__v1z.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@hash__v2-2.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@norm__v1z.snap create mode 100644 mbtiles/tests/snapshots/copy__databases@norm__v2-2.snap diff --git a/Cargo.lock b/Cargo.lock index 50ee96887..4e8e8de75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2401,7 +2401,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.10.0" +version = "0.11.0" dependencies = [ "actix-rt", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 5500b6e8b..c11682199 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ lambda-web = { version = "0.2.1", features = ["actix4"] } libsqlite3-sys = { version = ">=0.27", features = ["bundled"] } log = "0.4" martin-tile-utils = { path = "./martin-tile-utils", version = "0.5.0" } -mbtiles = { path = "./mbtiles", version = "0.10.0" } +mbtiles = { path = "./mbtiles", version = "0.11.0" } md5 = "0.7.0" moka = { version = "0.12", features = ["future"] } num_cpus = "1" diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index d56fbdee8..c0cee0bda 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.10.0" +version = "0.11.0" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 7093c2bbd..54b800d46 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use log::error; use mbtiles::{ apply_patch, AggHashType, CopyDuplicateMode, CopyType, IntegrityCheckType, MbtResult, - MbtTypeCli, Mbtiles, MbtilesCopier, PatchType, UpdateZoomType, + MbtTypeCli, Mbtiles, MbtilesCopier, PatchTypeCli, UpdateZoomType, }; use tilejson::Bounds; @@ -116,8 +116,8 @@ pub struct CopyArgs { #[arg(long, conflicts_with("diff_with_file"))] apply_patch: Option, /// Specify the type of patch file to generate. - #[arg(long, requires("diff_with_file"), default_value_t=PatchType::default())] - patch_type: PatchType, + #[arg(long, requires("diff_with_file"), default_value_t=PatchTypeCli::default())] + patch_type: PatchTypeCli, } #[allow(clippy::doc_markdown)] @@ -130,8 +130,8 @@ pub struct DiffArgs { /// Output file to write the resulting difference to diff: PathBuf, /// Specify the type of patch file to generate. - #[arg(long, default_value_t=PatchType::default())] - patch_type: PatchType, + #[arg(long, default_value_t=PatchTypeCli::default())] + patch_type: PatchTypeCli, #[command(flatten)] pub options: SharedCopyOpts, @@ -181,12 +181,12 @@ impl SharedCopyOpts { dst_file: PathBuf, diff_with_file: Option, apply_patch: Option, - patch_type: PatchType, + patch_type: PatchTypeCli, ) -> MbtilesCopier { MbtilesCopier { src_file, dst_file, - diff_with_file: diff_with_file.map(|p| (p, patch_type)), + diff_with_file: diff_with_file.map(|p| (p, patch_type.into())), apply_patch, // Shared copy: self.copy, @@ -329,7 +329,6 @@ mod tests { use clap::error::ErrorKind; use clap::Parser; use mbtiles::CopyDuplicateMode; - use mbtiles::PatchType::Whole; use super::*; use crate::Commands::{ApplyPatch, Copy, Diff, MetaGetValue, MetaSetValue, Validate}; @@ -540,7 +539,7 @@ mod tests { file1: PathBuf::from("file1.mbtiles"), file2: PathBuf::from("file2.mbtiles"), diff: PathBuf::from("../delta.mbtiles"), - patch_type: Whole, + patch_type: PatchTypeCli::Whole, options: SharedCopyOpts { on_duplicate: Some(CopyDuplicateMode::Override), ..Default::default() diff --git a/mbtiles/src/bindiff.rs b/mbtiles/src/bindiff.rs index e6ad0dbe3..0ba3f33b7 100644 --- a/mbtiles/src/bindiff.rs +++ b/mbtiles/src/bindiff.rs @@ -4,19 +4,51 @@ use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; use std::time::Instant; +use enum_display::EnumDisplay; use flume::{bounded, Receiver, Sender}; use futures::TryStreamExt; use log::{debug, error, info}; use martin_tile_utils::{decode_brotli, decode_gzip, encode_brotli, encode_gzip, TileCoord}; +use serde::{Deserialize, Serialize}; use sqlite_compressions::{BsdiffRawDiffer, Differ as _}; use sqlx::{query, Executor, Row, SqliteConnection}; use xxhash_rust::xxh3::xxh3_64; use crate::MbtType::{Flat, FlatWithHash, Normalized}; -use crate::PatchType::Whole; -use crate::{ - create_bsdiffraw_tables, get_bsdiff_tbl_name, MbtError, MbtResult, MbtType, Mbtiles, PatchType, -}; +use crate::PatchType::{BinDiffGz, BinDiffRaw}; +use crate::{create_bsdiffraw_tables, get_bsdiff_tbl_name, MbtError, MbtResult, MbtType, Mbtiles}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] +#[enum_display(case = "Kebab")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum PatchTypeCli { + /// Patch file will contain the entire tile if it is different from the source + #[default] + Whole, + /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as gzipped blobs, decoding them before diffing. + BinDiffGz, + /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as blobs without any special encoding. + BinDiffRaw, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] +#[enum_display(case = "Kebab")] +pub enum PatchType { + /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as gzipped blobs, decoding them before diffing. + BinDiffGz, + /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as blobs without any special encoding. + BinDiffRaw, +} + +impl From for Option { + fn from(cli: PatchTypeCli) -> Self { + match cli { + PatchTypeCli::Whole => None, + PatchTypeCli::BinDiffGz => Some(BinDiffGz), + PatchTypeCli::BinDiffRaw => Some(BinDiffRaw), + } + } +} pub trait BinDiffer: Sized + Send + Sync + 'static { fn query( @@ -153,7 +185,6 @@ impl BinDiffDiffer { dif_type: MbtType, patch_type: PatchType, ) -> Self { - assert_ne!(patch_type, Whole, "Invalid for BinDiffDiffer"); let insert_sql = format!( "INSERT INTO {}(zoom_level, tile_column, tile_row, patch_data, tile_xxh3_64_hash) VALUES (?, ?, ?, ?, ?)", get_bsdiff_tbl_name(patch_type)); @@ -219,7 +250,7 @@ impl BinDiffer for BinDiffDiffer { fn process(&self, value: DifferBefore) -> MbtResult { let mut old_tile = value.old_tile_data; let mut new_tile = value.new_tile_data; - if self.patch_type == PatchType::BinDiffGz { + if self.patch_type == BinDiffGz { old_tile = decode_gzip(&old_tile).inspect_err(|e| { error!("Unable to gzip-decode source tile {:?}: {e}", value.coord); })?; @@ -258,14 +289,14 @@ impl BinDiffer for BinDiffDiffer { pub struct ApplierBefore { coord: TileCoord, - tile_data: Vec, + old_tile: Vec, patch_data: Vec, uncompressed_tile_hash: u64, } pub struct ApplierAfter { coord: TileCoord, - data: Vec, + new_tile: Vec, new_tile_hash: String, } @@ -322,7 +353,7 @@ impl BinDiffer for BinDiffPatcher { x: row.get(1), y: row.get(2), }, - tile_data: row.get(3), + old_tile: row.get(3), patch_data: row.get(4), #[allow(clippy::cast_sign_loss)] uncompressed_tile_hash: row.get::(5) as u64, @@ -336,12 +367,21 @@ impl BinDiffer for BinDiffPatcher { } fn process(&self, value: ApplierBefore) -> MbtResult { - let tile_data = decode_gzip(&value.tile_data) - .inspect_err(|e| error!("Unable to gzip-decode source tile {:?}: {e}", value.coord))?; + let old_tile = if self.patch_type == BinDiffGz { + decode_gzip(&value.old_tile).inspect_err(|e| { + error!("Unable to gzip-decode source tile {:?}: {e}", value.coord); + })? + } else { + value.old_tile + }; + let patch_data = decode_brotli(&value.patch_data) .inspect_err(|e| error!("Unable to brotli-decode patch data {:?}: {e}", value.coord))?; - let new_tile = BsdiffRawDiffer::patch(&tile_data, &patch_data) + + let mut new_tile = BsdiffRawDiffer::patch(&old_tile, &patch_data) .inspect_err(|e| error!("Unable to patch tile {:?}: {e}", value.coord))?; + + // Verify the hash of the patched tile is what we expect let new_tile_hash = xxh3_64(&new_tile); if new_tile_hash != value.uncompressed_tile_hash { return Err(MbtError::BinDiffIncorrectTileHash( @@ -351,16 +391,18 @@ impl BinDiffer for BinDiffPatcher { )); } - let data = encode_gzip(&new_tile)?; + if self.patch_type == BinDiffGz { + new_tile = encode_gzip(&new_tile)?; + }; Ok(ApplierAfter { coord: value.coord, new_tile_hash: if self.dst_type == FlatWithHash { - format!("{:X}", md5::compute(&data)) + format!("{:X}", md5::compute(&new_tile)) } else { String::default() // This is a fast noop, no memory alloc is performed }, - data, + new_tile, }) } @@ -378,7 +420,7 @@ impl BinDiffer for BinDiffPatcher { .bind(value.coord.z) .bind(value.coord.x) .bind(value.coord.y) - .bind(value.data); + .bind(value.new_tile); if self.dst_type == FlatWithHash { q = q.bind(value.new_tile_hash); diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index e87c76ecc..c3200c350 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -10,7 +10,8 @@ use sqlite_hashes::rusqlite::Connection; use sqlx::{query, Connection as _, Executor as _, Row, SqliteConnection}; use tilejson::Bounds; -use crate::bindiff::{BinDiffDiffer, BinDiffPatcher, BinDiffer as _}; +use crate::bindiff::PatchType::BinDiffGz; +use crate::bindiff::{BinDiffDiffer, BinDiffPatcher, BinDiffer as _, PatchType}; use crate::errors::MbtResult; use crate::mbtiles::PatchFileInfo; use crate::queries::{ @@ -19,7 +20,7 @@ use crate::queries::{ use crate::AggHashType::Verify; use crate::IntegrityCheckType::Quick; use crate::MbtType::{Flat, FlatWithHash, Normalized}; -use crate::PatchType::{BinDiffGz, BinDiffRaw, Whole}; +use crate::PatchType::BinDiffRaw; use crate::{ action_with_rusqlite, get_bsdiff_tbl_name, invert_y_value, reset_db_settings, AggHashType, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_AFTER_APPLY, @@ -35,19 +36,6 @@ pub enum CopyDuplicateMode { Abort, } -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] -#[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] -pub enum PatchType { - /// Patch file will contain the entire tile if it is different from the source - #[default] - Whole, - /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as gzipped blobs, decoding them before diffing. - BinDiffGz, - /// Use bin-diff to store only the bytes changed between two versions of each tile. Treats content as blobs without any special encoding. - BinDiffRaw, -} - impl CopyDuplicateMode { #[must_use] pub fn to_sql(&self) -> &'static str { @@ -82,7 +70,7 @@ pub struct MbtilesCopier { /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. pub bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. Also specifies the type of patch to generate. - pub diff_with_file: Option<(PathBuf, PatchType)>, + pub diff_with_file: Option<(PathBuf, Option)>, /// Apply a patch file while copying src to dst. pub apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. @@ -213,7 +201,7 @@ impl MbtileCopierInt { async fn run_with_diff( self, dif_mbt: Mbtiles, - patch_type: PatchType, + patch_type: Option, ) -> MbtResult { let mut dif_conn = dif_mbt.open_readonly().await?; let dif_info = dif_mbt.examine_diff(&mut dif_conn).await?; @@ -231,7 +219,7 @@ impl MbtileCopierInt { dif_mbt.attach_to(&mut conn, "diffDb").await?; let dst_type = self.options.dst_type().unwrap_or(src_info.mbt_type); - if patch_type != Whole && matches!(dst_type, Normalized { .. }) { + if patch_type.is_some() && matches!(dst_type, Normalized { .. }) { return Err(MbtError::BinDiffRequiresFlatWithHash(dst_type)); } @@ -243,11 +231,7 @@ impl MbtileCopierInt { dif_type = dif_info.mbt_type, what = self.copy_text(), dst_path = self.dst_mbt.filepath(), - patch = match patch_type { - Whole => {""} - BinDiffGz => {" with bin-diff"} - BinDiffRaw => {" with bin-diff-raw"} - } + patch = patch_type_str(patch_type), ); self.init_schema(&mut conn, src_info.mbt_type, dst_type) @@ -265,7 +249,7 @@ impl MbtileCopierInt { detach_db(&mut conn, "diffDb").await?; detach_db(&mut conn, "sourceDb").await?; - if patch_type != Whole { + if let Some(patch_type) = patch_type { BinDiffDiffer::new(self.src_mbt.clone(), dif_mbt, dif_info.mbt_type, patch_type) .run(&mut conn, self.get_where_clause("srcTiles.")) .await?; @@ -302,7 +286,7 @@ impl MbtileCopierInt { let src_type = self.validate_src_file().await?.mbt_type; let dst_type = self.options.dst_type().unwrap_or(src_type); - if dif_info.patch_type != Whole && matches!(dst_type, Normalized { .. }) { + if dif_info.patch_type.is_some() && matches!(dst_type, Normalized { .. }) { return Err(MbtError::BinDiffRequiresFlatWithHash(dst_type)); } @@ -320,11 +304,7 @@ impl MbtileCopierInt { src_mbt = self.src_mbt, what = self.copy_text(), dst_path = self.dst_mbt.filepath(), - patch = match dif_info.patch_type { - Whole => {""} - BinDiffGz => {" with bin-diff"} - BinDiffRaw => {" with bin-diff-raw"} - } + patch = patch_type_str(dif_info.patch_type), ); self.init_schema(&mut conn, src_type, dst_type).await?; @@ -339,21 +319,16 @@ impl MbtileCopierInt { detach_db(&mut conn, "diffDb").await?; detach_db(&mut conn, "sourceDb").await?; - if dif_info.patch_type != Whole { - BinDiffPatcher::new( - self.src_mbt.clone(), - dif_mbt.clone(), - dst_type, - dif_info.patch_type, - ) - .run(&mut conn, self.get_where_clause("srcTiles.")) - .await?; + if let Some(patch_type) = dif_info.patch_type { + BinDiffPatcher::new(self.src_mbt.clone(), dif_mbt.clone(), dst_type, patch_type) + .run(&mut conn, self.get_where_clause("srcTiles.")) + .await?; } // TODO: perhaps disable all except --copy all when using with diffs, or else is not making much sense if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash { self.dst_mbt.update_agg_tiles_hash(&mut conn).await?; - if dif_info.patch_type == BinDiffGz { + if matches!(dif_info.patch_type, Some(BinDiffGz)) { info!("Skipping {AGG_TILES_HASH_AFTER_APPLY} validation because re-gzip-ing could produce different tile data. Each bindiff-ed tile was still verified with a hash value"); } else { let new_hash = self.dst_mbt.get_agg_tiles_hash(&mut conn).await?; @@ -375,11 +350,12 @@ impl MbtileCopierInt { } } - let hash_type = if dif_info.patch_type == BinDiffGz || self.options.skip_agg_tiles_hash { - AggHashType::Off - } else { - Verify - }; + let hash_type = + if matches!(dif_info.patch_type, Some(BinDiffGz)) || self.options.skip_agg_tiles_hash { + AggHashType::Off + } else { + Verify + }; if self.options.validate { self.dst_mbt.validate(&mut conn, Quick, hash_type).await?; @@ -731,11 +707,9 @@ fn get_select_from_apply_patch( let src_tiles = query_for_dst("sourceDb", src_type, dst_type); let diff_tiles = query_for_dst("diffDb", dif_info.mbt_type, dst_type); - let (bindiff_from, bindiff_cond) = if dif_info.patch_type == Whole { - (String::new(), "") - } else { + let (bindiff_from, bindiff_cond) = if let Some(patch_type) = dif_info.patch_type { // do not copy any tiles that are in the patch table - let tbl = get_bsdiff_tbl_name(dif_info.patch_type); + let tbl = get_bsdiff_tbl_name(patch_type); ( format!( " @@ -746,6 +720,8 @@ fn get_select_from_apply_patch( ), "AND bdTbl.patch_data ISNULL", ) + } else { + (String::new(), "") }; // Take dif tile_data if it is set, otherwise take the one from src @@ -769,7 +745,7 @@ fn get_select_from_apply_patch( fn get_select_from_with_diff( dif_type: MbtType, dst_type: MbtType, - patch_type: PatchType, + patch_type: Option, ) -> String { let tile_hash_expr; let diff_tiles; @@ -792,10 +768,10 @@ fn get_select_from_with_diff( }; } - let sql_cond = if patch_type == Whole { - "OR srcTiles.tile_data != difTiles.tile_data" - } else { + let sql_cond = if patch_type.is_some() { "" + } else { + "OR srcTiles.tile_data != difTiles.tile_data" }; format!( " @@ -842,6 +818,17 @@ fn get_select_from(src_type: MbtType, dst_type: MbtType) -> &'static str { } } +fn patch_type_str(patch_type: Option) -> &'static str { + if let Some(v) = patch_type { + match v { + BinDiffGz => " with bin-diff on gzip-ed tiles", + BinDiffRaw => " with bin-diff-raw", + } + } else { + "" + } +} + #[cfg(test)] mod tests { use sqlx::{Decode, Sqlite, SqliteConnection, Type}; @@ -1022,7 +1009,7 @@ mod tests { let opt = MbtilesCopier { src_file: src.clone(), dst_file: dst.clone(), - diff_with_file: Some((diff_file.clone(), Whole)), + diff_with_file: Some((diff_file.clone(), None)), force: true, ..Default::default() }; diff --git a/mbtiles/src/errors.rs b/mbtiles/src/errors.rs index 9009d1987..07b30fe38 100644 --- a/mbtiles/src/errors.rs +++ b/mbtiles/src/errors.rs @@ -102,7 +102,7 @@ pub enum MbtError { #[error("Applying bindiff to tile {0} resulted in mismatching hash: expecting `{1}` != computed uncompressed value `{2}`")] BinDiffIncorrectTileHash(String, String, String), - #[error("Internal error creating bin-diff table")] + #[error("Unable to generate or apply bin-diff patch")] BindiffError, #[error("BinDiff patch files can be only applied with `mbtiles copy --apply-patch` command")] diff --git a/mbtiles/src/lib.rs b/mbtiles/src/lib.rs index 6359a2601..4de855359 100644 --- a/mbtiles/src/lib.rs +++ b/mbtiles/src/lib.rs @@ -1,10 +1,11 @@ #![doc = include_str!("../README.md")] // Re-export sqlx +pub use bindiff::{PatchType, PatchTypeCli}; pub use sqlx; mod copier; -pub use copier::{CopyDuplicateMode, MbtilesCopier, PatchType}; +pub use copier::{CopyDuplicateMode, MbtilesCopier}; mod errors; pub use errors::{MbtError, MbtResult}; diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index 04323cafb..c0cc920fa 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -10,8 +10,9 @@ use sqlite_hashes::register_md5_functions; use sqlx::sqlite::SqliteConnectOptions; use sqlx::{query, Connection as _, Executor, SqliteConnection, SqliteExecutor, Statement}; +use crate::bindiff::PatchType; use crate::errors::{MbtError, MbtResult}; -use crate::{invert_y_value, CopyDuplicateMode, MbtType, PatchType}; +use crate::{invert_y_value, CopyDuplicateMode, MbtType}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] #[enum_display(case = "Kebab")] @@ -48,7 +49,7 @@ pub struct PatchFileInfo { pub agg_tiles_hash: Option, pub agg_tiles_hash_before_apply: Option, pub agg_tiles_hash_after_apply: Option, - pub patch_type: PatchType, + pub patch_type: Option, } #[derive(Clone, Debug)] diff --git a/mbtiles/src/patcher.rs b/mbtiles/src/patcher.rs index 163b5f7fb..02f89fcb5 100644 --- a/mbtiles/src/patcher.rs +++ b/mbtiles/src/patcher.rs @@ -5,7 +5,6 @@ use sqlx::{query, Connection as _}; use crate::queries::detach_db; use crate::MbtType::{Flat, FlatWithHash, Normalized}; -use crate::PatchType::Whole; use crate::{ MbtError, MbtResult, MbtType, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_AFTER_APPLY, AGG_TILES_HASH_BEFORE_APPLY, @@ -17,7 +16,7 @@ pub async fn apply_patch(base_file: PathBuf, patch_file: PathBuf, force: bool) - let mut conn = patch_mbt.open_readonly().await?; let patch_info = patch_mbt.examine_diff(&mut conn).await?; - if patch_info.patch_type != Whole { + if patch_info.patch_type.is_some() { return Err(MbtError::UnsupportedPatchType); } patch_mbt.validate_diff_info(&patch_info, force)?; diff --git a/mbtiles/src/queries.rs b/mbtiles/src/queries.rs index 7443a406e..427ccc1d3 100644 --- a/mbtiles/src/queries.rs +++ b/mbtiles/src/queries.rs @@ -3,9 +3,10 @@ use martin_tile_utils::MAX_ZOOM; use sqlite_compressions::rusqlite::Connection; use sqlx::{query, Executor as _, Row, SqliteConnection, SqliteExecutor}; +use crate::bindiff::PatchType; use crate::errors::MbtResult; use crate::MbtError::InvalidZoomValue; -use crate::{MbtType, PatchType}; +use crate::MbtType; /// Returns true if the database is empty (no tables/indexes/...) pub async fn is_empty_database(conn: &mut T) -> MbtResult @@ -215,7 +216,6 @@ pub fn get_bsdiff_tbl_name(patch_type: PatchType) -> &'static str { match patch_type { PatchType::BinDiffRaw => "bsdiffraw", PatchType::BinDiffGz => "bsdiffrawgz", - PatchType::Whole => panic!("Unexpected PatchType::Whole"), } } @@ -241,7 +241,7 @@ where /// Check if `MBTiles` has a table or a view named `bsdiffraw` or `bsdiffrawgz` with needed fields, /// and return the corresponding patch type. If missing, return `PatchType::Whole` -pub async fn get_patch_type(conn: &mut T) -> MbtResult +pub async fn get_patch_type(conn: &mut T) -> MbtResult> where for<'e> &'e mut T: SqliteExecutor<'e>, { @@ -272,11 +272,11 @@ where .unwrap_or_default() == 1 { - return Ok(pt); + return Ok(Some(pt)); } } - Ok(PatchType::Whole) + Ok(None) } pub async fn create_normalized_tables(conn: &mut T) -> MbtResult<()> diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 414919aa6..f64140a86 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -11,10 +11,10 @@ use martin_tile_utils::xyz_to_bbox; use mbtiles::AggHashType::Verify; use mbtiles::IntegrityCheckType::Off; use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; -use mbtiles::PatchType::{BinDiffRaw, Whole}; +use mbtiles::PatchTypeCli::{BinDiffGz, BinDiffRaw}; use mbtiles::{ apply_patch, init_mbtiles_schema, invert_y_value, CopyType, MbtResult, MbtTypeCli, Mbtiles, - MbtilesCopier, PatchType, UpdateZoomType, + MbtilesCopier, PatchTypeCli, UpdateZoomType, }; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; @@ -22,6 +22,8 @@ use serde::Serialize; use sqlx::{query, query_as, Executor as _, Row, SqliteConnection}; use tokio::runtime::Handle; +const GZIP_TILES: &str = "UPDATE tiles SET tile_data = gzip(tile_data);"; + const TILES_V1: &str = " INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES --(z, x, y, data) -- rules: keep if x=0, edit if x=1, remove if x=2 @@ -138,21 +140,28 @@ macro_rules! open { /// Create a new `SQLite` file of given type without `agg_tiles_hash` metadata value macro_rules! new_file_no_hash { ($function:ident, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ - new_file!(@true, $function, $dst_type_cli, $sql_meta, $sql_data, $($arg)*) + new_file!(@true, $function, $dst_type_cli, $sql_meta, $sql_data, "", $($arg)*) }}; } /// Create a new `SQLite` file of type `$dst_type_cli` with the given metadata and tiles macro_rules! new_file { ($function:ident, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => { - new_file!(@false, $function, $dst_type_cli, $sql_meta, $sql_data, $($arg)*) + new_file!(@false, $function, $dst_type_cli, $sql_meta, $sql_data, "", $($arg)*) + }; + + (+ $sql:expr, $function:ident, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => { + new_file!(@false, $function, $dst_type_cli, $sql_meta, $sql_data, $sql, $($arg)*) }; - (@$skip_agg:expr, $function:tt, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ + (@ $skip_agg:expr, $function:tt, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $sql:expr, $($arg:tt)*) => {{ let (tmp_mbt, mut cn_tmp) = open!(@"temp", $function, $($arg)*); init_mbtiles_schema(&mut cn_tmp, mbtiles::MbtType::Flat).await.unwrap(); cn_tmp.execute($sql_data).await.unwrap(); cn_tmp.execute($sql_meta).await.unwrap(); + if $sql != "" { + cn_tmp.execute($sql).await.unwrap(); + } let (dst_mbt, cn_dst) = open!($function, $($arg)*); copy! { @@ -292,7 +301,7 @@ fn databases() -> Databases { copy! { result.path("v1", mbt_typ), path(&dif_mbt), - diff_with_file => Some((result.path("v2", mbt_typ), Whole)), + diff_with_file => Some((result.path("v2", mbt_typ), None)), }; let dmp = dump(&mut dif_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__dif"); @@ -302,21 +311,60 @@ fn databases() -> Databases { } result.add("dif", mbt_typ, dmp, dif_mbt, Some(hash), dif_cn); - // ----------------- bdr (v1 -> v2) -- bin-diff-raw ----------------- + // ----------------- v1z ----------------- + let (v1z_mbt, mut v1z_cn) = + new_file!(+GZIP_TILES, databases, mbt_typ, METADATA_V1, TILES_V1, "{typ}__v1z"); + let dmp = dump(&mut v1z_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__v1z"); + let hash = v1z_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"C0CA886B149CE416242AB2AFE8E641AD"); + } + result.add("v1z", mbt_typ, dmp, v1z_mbt, Some(hash), v1z_cn); + + // ----------------- v2z ----------------- + let (v2z_mbt, mut v2z_cn) = + new_file!(+GZIP_TILES, databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2z"); + let dmp = dump(&mut v2z_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__v2"); + let hash = v2z_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"A18D0C39730FB52E5A547F096F5C60E8"); + } + result.add("v2z", mbt_typ, dmp, v2z_mbt, Some(hash), v2z_cn); + + // ----------------- bin-diff (v1 -> v2) ----------------- if mbt_typ == Flat || mbt_typ == FlatWithHash { - let (bdr_mbt, mut bdr_cn) = open!(databases, "{typ}__bdr"); - copy! { - result.path("v1", mbt_typ), - path(&bdr_mbt), - diff_with_file => Some((result.path("v2", mbt_typ), BinDiffRaw)), - }; - let dmp = dump(&mut bdr_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__bdr"); - let hash = bdr_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"585A88FEEC740448FF1EB4F96088FFE3"); + for (a, b, patch_type, pt) in [ + ("v1", "v2", BinDiffRaw, "bdr"), + ("v1z", "v2z", BinDiffGz, "bdz"), + ] { + let (bd_mbt, mut bd_cn) = open!(databases, "{typ}__{pt}"); + copy! { + result.path(a, mbt_typ), + path(&bd_mbt), + diff_with_file => Some((result.path(b, mbt_typ), patch_type.into())), + }; + let dmp = dump(&mut bd_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__{pt}"); + let hash = bd_mbt.open_and_validate(Off, Verify).await.unwrap(); + match patch_type { + PatchTypeCli::Whole => { + unreachable!() + } + BinDiffGz => { + allow_duplicates!( + assert_snapshot!(hash, @"9AFEC3326B465CB939664C47A572D4C6") + ); + } + BinDiffRaw => { + allow_duplicates!( + assert_snapshot!(hash, @"585A88FEEC740448FF1EB4F96088FFE3") + ); + } + } + result.add(pt, mbt_typ, dmp, bd_mbt, Some(hash), bd_cn); } - result.add("bdr", mbt_typ, dmp, bdr_mbt, Some(hash), bdr_cn); } // ----------------- v1_clone ----------------- @@ -340,7 +388,7 @@ fn databases() -> Databases { copy! { result.path("v1", mbt_typ), path(&dif_empty_mbt), - diff_with_file => Some((result.path("v1_clone", mbt_typ), Whole)), + diff_with_file => Some((result.path("v1_clone", mbt_typ), None)), }; let dmp = dump(&mut dif_empty_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__dif_empty"); @@ -491,7 +539,7 @@ async fn diff_and_patch( copy! { databases.path(a_db, a_type), path(&dif_mbt), - diff_with_file => Some((databases.path(b_db, b_type), Whole)), + diff_with_file => Some((databases.path(b_db, b_type), None)), dst_type_cli => dif_type, }; pretty_assert_eq!( @@ -533,12 +581,15 @@ async fn diff_and_patch_bsdiff( #[values(Flat, FlatWithHash)] a_type: MbtTypeCli, #[values(Flat, FlatWithHash)] b_type: MbtTypeCli, #[values(Flat, FlatWithHash)] dif_type: MbtTypeCli, - #[values(BinDiffRaw)] patch_type: PatchType, #[values(Flat, FlatWithHash)] dst_type: MbtTypeCli, - #[values(("v1", "v2", "bdr"))] tilesets: (&'static str, &'static str, &'static str), + #[values( + ("v1", "v2", "bdr", BinDiffRaw), + ("v1z", "v2z", "bdz", BinDiffGz), + )] + tilesets: (&'static str, &'static str, &'static str, PatchTypeCli), #[notrace] databases: &Databases, ) -> MbtResult<()> { - let (a_db, b_db, dif_db) = tilesets; + let (a_db, b_db, dif_db, patch_type) = tilesets; let dif = shorten(dif_type); let prefix = format!( "{a_db}_{}--{b_db}_{}={dif}_{patch_type}", @@ -551,7 +602,7 @@ async fn diff_and_patch_bsdiff( copy! { databases.path(a_db, a_type), path(&dif_mbt), - diff_with_file => Some((databases.path(b_db, b_type), patch_type)), + diff_with_file => Some((databases.path(b_db, b_type), patch_type.into())), dst_type_cli => Some(dif_type), }; pretty_assert_eq!(&dump(&mut dif_cn).await?, databases.dump(dif_db, dif_type)); @@ -624,9 +675,8 @@ async fn test_one() { FlatWithHash, FlatWithHash, FlatWithHash, - BinDiffRaw, FlatWithHash, - ("v1", "v2", "bdr"), + ("v1", "v2", "bdr", BinDiffRaw), &db, ) .await diff --git a/mbtiles/tests/snapshots/copy__databases@flat__bdz.snap b/mbtiles/tests/snapshots/copy__databases@flat__bdz.snap new file mode 100644 index 000000000..4b2071214 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@flat__bdz.snap @@ -0,0 +1,67 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'bsdiffrawgz' +sql = ''' +CREATE TABLE bsdiffrawgz ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + patch_data blob NOT NULL, + tile_xxh3_64_hash integer NOT NULL, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(1B1E00F80799700B0AE100F45284A210A00708A0C03B), 479130493 )', + '( 5, 1, 2, blob(1B2000F8079970D30C62EDF2D8285E11400000BB2F01), -1097843426 )', + '( 5, 1, 3, blob(;), 953390274 )', + '( 6, 1, 4, blob(1B1F00F8077163E37063303653B324A12804E8010450E0ED01), 386481748 )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9AFEC3326B465CB939664C47A572D4C6" )', + '( "agg_tiles_hash_after_apply", "A18D0C39730FB52E5A547F096F5C60E8" )', + '( "agg_tiles_hash_before_apply", "C0CA886B149CE416242AB2AFE8E641AD" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-remove", NULL )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 2, 2, NULL )', + '( 5, 2, 3, NULL )', + '( 5, 3, 7, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000) )', + '( 5, 3, 8, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000) )', + '( 6, 2, 6, NULL )', +] + +[[]] +type = 'index' +tbl_name = 'bsdiffrawgz' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__databases@flat__v1z.snap b/mbtiles/tests/snapshots/copy__databases@flat__v1z.snap new file mode 100644 index 000000000..852880035 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@flat__v1z.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "C0CA886B149CE416242AB2AFE8E641AD" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000) )', + '( 5, 0, 0, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( 5, 0, 1, blob(1F8B08000000000000FF03000000000000000000) )', + '( 5, 1, 1, blob(1F8B08000000000000FF4B4DC92CD12D3304007ED8D6BF07000000) )', + '( 5, 1, 2, blob(1F8B08000000000000FF03000000000000000000) )', + '( 5, 1, 3, blob(1F8B08000000000000FFCBCBCFD34DCD2D28A9040006173DB509000000) )', + '( 5, 2, 2, blob(1F8B08000000000000FF2B4ACDCD2F4B0500301D806806000000) )', + '( 5, 2, 3, blob(1F8B08000000000000FF03000000000000000000) )', + '( 6, 0, 3, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( 6, 0, 5, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000) )', + '( 6, 1, 4, blob(1F8B08000000000000FF4B4DC92CD12D3304007ED8D6BF07000000) )', + '( 6, 2, 6, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__databases@flat__v2-2.snap b/mbtiles/tests/snapshots/copy__databases@flat__v2-2.snap new file mode 100644 index 000000000..ee2ba48ef --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@flat__v2-2.snap @@ -0,0 +1,49 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "A18D0C39730FB52E5A547F096F5C60E8" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000) )', + '( 5, 0, 0, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( 5, 0, 1, blob(1F8B08000000000000FF03000000000000000000) )', + '( 5, 1, 1, blob(1F8B08000000000000FF4B4DC92CD12D330200C489DF2607000000) )', + '( 5, 1, 2, blob(1F8B08000000000000FFCBCB2FD14DCD2D28A9040086D4937609000000) )', + '( 5, 1, 3, blob(1F8B08000000000000FF03000000000000000000) )', + '( 5, 3, 7, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000) )', + '( 5, 3, 8, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000) )', + '( 6, 0, 3, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( 6, 0, 5, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000) )', + '( 6, 1, 4, blob(1F8B08000000000000FF4B4DC92CD12D334A04006367987408000000) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__databases@hash__bdz.snap b/mbtiles/tests/snapshots/copy__databases@hash__bdz.snap new file mode 100644 index 000000000..3f24877f1 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@hash__bdz.snap @@ -0,0 +1,75 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'bsdiffrawgz' +sql = ''' +CREATE TABLE bsdiffrawgz ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + patch_data blob NOT NULL, + tile_xxh3_64_hash integer NOT NULL, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(1B1E00F80799700B0AE100F45284A210A00708A0C03B), 479130493 )', + '( 5, 1, 2, blob(1B2000F8079970D30C62EDF2D8285E11400000BB2F01), -1097843426 )', + '( 5, 1, 3, blob(;), 953390274 )', + '( 6, 1, 4, blob(1B1F00F8077163E37063303653B324A12804E8010450E0ED01), 386481748 )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9AFEC3326B465CB939664C47A572D4C6" )', + '( "agg_tiles_hash_after_apply", "A18D0C39730FB52E5A547F096F5C60E8" )', + '( "agg_tiles_hash_before_apply", "C0CA886B149CE416242AB2AFE8E641AD" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-remove", NULL )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 2, 2, NULL, "" )', + '( 5, 2, 3, NULL, "" )', + '( 5, 3, 7, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000), "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 5, 3, 8, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000), "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 6, 2, 6, NULL, "" )', +] + +[[]] +type = 'index' +tbl_name = 'bsdiffrawgz' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__databases@hash__v1z.snap b/mbtiles/tests/snapshots/copy__databases@hash__v1z.snap new file mode 100644 index 000000000..107aade19 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@hash__v1z.snap @@ -0,0 +1,58 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "C0CA886B149CE416242AB2AFE8E641AD" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000), "0A62AF3A2A3D38C0E8FF098A684C3EC1" )', + '( 5, 0, 0, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000), "98AD49106F1CE0AA003027C229A70F7E" )', + '( 5, 0, 1, blob(1F8B08000000000000FF03000000000000000000), "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 1, blob(1F8B08000000000000FF4B4DC92CD12D3304007ED8D6BF07000000), "A4D55DDE3B49D78DD9846688A0786F2D" )', + '( 5, 1, 2, blob(1F8B08000000000000FF03000000000000000000), "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 3, blob(1F8B08000000000000FFCBCBCFD34DCD2D28A9040006173DB509000000), "4DE1D83BFD7CB6ACE583AA7D5A18725A" )', + '( 5, 2, 2, blob(1F8B08000000000000FF2B4ACDCD2F4B0500301D806806000000), "0D9BABF1C0099632D55F6274FB15419F" )', + '( 5, 2, 3, blob(1F8B08000000000000FF03000000000000000000), "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 6, 0, 3, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000), "98AD49106F1CE0AA003027C229A70F7E" )', + '( 6, 0, 5, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000), "3CD93E79D6812F995906036D24282DBE" )', + '( 6, 1, 4, blob(1F8B08000000000000FF4B4DC92CD12D3304007ED8D6BF07000000), "A4D55DDE3B49D78DD9846688A0786F2D" )', + '( 6, 2, 6, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000), "3CD93E79D6812F995906036D24282DBE" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__databases@hash__v2-2.snap b/mbtiles/tests/snapshots/copy__databases@hash__v2-2.snap new file mode 100644 index 000000000..072281bbb --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@hash__v2-2.snap @@ -0,0 +1,57 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "A18D0C39730FB52E5A547F096F5C60E8" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000), "0A62AF3A2A3D38C0E8FF098A684C3EC1" )', + '( 5, 0, 0, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000), "98AD49106F1CE0AA003027C229A70F7E" )', + '( 5, 0, 1, blob(1F8B08000000000000FF03000000000000000000), "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 1, blob(1F8B08000000000000FF4B4DC92CD12D330200C489DF2607000000), "39794F1D8EEDEE7777FA89D4FD8D3154" )', + '( 5, 1, 2, blob(1F8B08000000000000FFCBCB2FD14DCD2D28A9040086D4937609000000), "94335B9F0CCBA1C1CB7F91B49EB34344" )', + '( 5, 1, 3, blob(1F8B08000000000000FF03000000000000000000), "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 3, 7, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000), "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 5, 3, 8, blob(1F8B08000000000000FFCB4B2D07004544E36B03000000), "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 6, 0, 3, blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000), "98AD49106F1CE0AA003027C229A70F7E" )', + '( 6, 0, 5, blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000), "3CD93E79D6812F995906036D24282DBE" )', + '( 6, 1, 4, blob(1F8B08000000000000FF4B4DC92CD12D334A04006367987408000000), "59A99A65DF08F16CE984BDFA6EBC95CF" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__databases@norm__v1z.snap b/mbtiles/tests/snapshots/copy__databases@norm__v1z.snap new file mode 100644 index 000000000..ea28bfbf5 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@norm__v1z.snap @@ -0,0 +1,97 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0A62AF3A2A3D38C0E8FF098A684C3EC1", blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000) )', + '( "0D9BABF1C0099632D55F6274FB15419F", blob(1F8B08000000000000FF2B4ACDCD2F4B0500301D806806000000) )', + '( "163BE0A88C70CA629FD516DBAADAD96A", blob(1F8B08000000000000FF03000000000000000000) )', + '( "3CD93E79D6812F995906036D24282DBE", blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000) )', + '( "4DE1D83BFD7CB6ACE583AA7D5A18725A", blob(1F8B08000000000000FFCBCBCFD34DCD2D28A9040006173DB509000000) )', + '( "98AD49106F1CE0AA003027C229A70F7E", blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( "A4D55DDE3B49D78DD9846688A0786F2D", blob(1F8B08000000000000FF4B4DC92CD12D3304007ED8D6BF07000000) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, "0A62AF3A2A3D38C0E8FF098A684C3EC1" )', + '( 5, 0, 0, "98AD49106F1CE0AA003027C229A70F7E" )', + '( 5, 0, 1, "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 1, "A4D55DDE3B49D78DD9846688A0786F2D" )', + '( 5, 1, 2, "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 3, "4DE1D83BFD7CB6ACE583AA7D5A18725A" )', + '( 5, 2, 2, "0D9BABF1C0099632D55F6274FB15419F" )', + '( 5, 2, 3, "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 6, 0, 3, "98AD49106F1CE0AA003027C229A70F7E" )', + '( 6, 0, 5, "3CD93E79D6812F995906036D24282DBE" )', + '( 6, 1, 4, "A4D55DDE3B49D78DD9846688A0786F2D" )', + '( 6, 2, 6, "3CD93E79D6812F995906036D24282DBE" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "C0CA886B149CE416242AB2AFE8E641AD" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__databases@norm__v2-2.snap b/mbtiles/tests/snapshots/copy__databases@norm__v2-2.snap new file mode 100644 index 000000000..e7dc362a3 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@norm__v2-2.snap @@ -0,0 +1,97 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0A62AF3A2A3D38C0E8FF098A684C3EC1", blob(1F8B08000000000000FF2BCACF2F01005BF9F41604000000) )', + '( "163BE0A88C70CA629FD516DBAADAD96A", blob(1F8B08000000000000FF03000000000000000000) )', + '( "39794F1D8EEDEE7777FA89D4FD8D3154", blob(1F8B08000000000000FF4B4DC92CD12D330200C489DF2607000000) )', + '( "3CD93E79D6812F995906036D24282DBE", blob(1F8B08000000000000FF33D4CD4E4D2DD035D42DCA050026D508C30B000000) )', + '( "59A99A65DF08F16CE984BDFA6EBC95CF", blob(1F8B08000000000000FF4B4DC92CD12D334A04006367987408000000) )', + '( "94335B9F0CCBA1C1CB7F91B49EB34344", blob(1F8B08000000000000FFCBCB2FD14DCD2D28A9040086D4937609000000) )', + '( "98AD49106F1CE0AA003027C229A70F7E", blob(1F8B08000000000000FF2B4ECC4D050044F150FC04000000) )', + '( "E1A151E7B18F8B2C53F94DF4CA201026", blob(1F8B08000000000000FFCB4B2D07004544E36B03000000) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, "0A62AF3A2A3D38C0E8FF098A684C3EC1" )', + '( 5, 0, 0, "98AD49106F1CE0AA003027C229A70F7E" )', + '( 5, 0, 1, "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 1, 1, "39794F1D8EEDEE7777FA89D4FD8D3154" )', + '( 5, 1, 2, "94335B9F0CCBA1C1CB7F91B49EB34344" )', + '( 5, 1, 3, "163BE0A88C70CA629FD516DBAADAD96A" )', + '( 5, 3, 7, "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 5, 3, 8, "E1A151E7B18F8B2C53F94DF4CA201026" )', + '( 6, 0, 3, "98AD49106F1CE0AA003027C229A70F7E" )', + '( 6, 0, 5, "3CD93E79D6812F995906036D24282DBE" )', + '( 6, 1, 4, "59A99A65DF08F16CE984BDFA6EBC95CF" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "A18D0C39730FB52E5A547F096F5C60E8" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/tests/expected/mbtiles/copy_bindiff.txt b/tests/expected/mbtiles/copy_bindiff.txt index 370e56264..dda1f87f4 100644 --- a/tests/expected/mbtiles/copy_bindiff.txt +++ b/tests/expected/mbtiles/copy_bindiff.txt @@ -1,3 +1,3 @@ -[INFO ] Comparing ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) and ./tests/fixtures/mbtiles/world_cities_modified.mbtiles (flat) into a new file tests/mbtiles_temp_files/world_cities_bindiff.mbtiles (flat) with bin-diff +[INFO ] Comparing ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) and ./tests/fixtures/mbtiles/world_cities_modified.mbtiles (flat) into a new file tests/mbtiles_temp_files/world_cities_bindiff.mbtiles (flat) with bin-diff on gzip-ed tiles [INFO ] Finished processing 1 bindiff tiles -[INFO ] Adding a new metadata value agg_tiles_hash = 8D0CA32D7634E1278476D3D22A0F7B01 in tests/mbtiles_temp_files/world_cities_bindiff.mbtiles +[INFO ] Adding a new metadata value agg_tiles_hash = 0A21AAF2C177B86DA3342A4F65794E49 in tests/mbtiles_temp_files/world_cities_bindiff.mbtiles diff --git a/tests/expected/mbtiles/copy_bindiff2.txt b/tests/expected/mbtiles/copy_bindiff2.txt index 5799b0491..7fbc3b8c5 100644 --- a/tests/expected/mbtiles/copy_bindiff2.txt +++ b/tests/expected/mbtiles/copy_bindiff2.txt @@ -1,5 +1,5 @@ [INFO ] The patch file tests/mbtiles_temp_files/world_cities_bindiff.mbtiles expects to be applied to a tileset with agg_tiles_hash=84792BF4EE9AEDDC5B1A60E707011FEE, and should result in hash 578FB5BD64746C39E3D344662947FD0D after applying -[INFO ] Applying patch from tests/mbtiles_temp_files/world_cities_bindiff.mbtiles (flat) to ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) into a new file tests/mbtiles_temp_files/world_cities_modified2.mbtiles (flat) with bin-diff +[INFO ] Applying patch from tests/mbtiles_temp_files/world_cities_bindiff.mbtiles (flat) to ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) into a new file tests/mbtiles_temp_files/world_cities_modified2.mbtiles (flat) with bin-diff on gzip-ed tiles [INFO ] Finished processing 1 bindiff tiles -[INFO ] Adding a new metadata value agg_tiles_hash = F863A28565B06142AED3E01EF8198CF5 in tests/mbtiles_temp_files/world_cities_modified2.mbtiles +[INFO ] Adding a new metadata value agg_tiles_hash = 623863EF20ABEFCB4E30EEFBD82FFFDC in tests/mbtiles_temp_files/world_cities_modified2.mbtiles [INFO ] Skipping agg_tiles_hash_after_apply validation because re-gzip-ing could produce different tile data. Each bindiff-ed tile was still verified with a hash value diff --git a/tests/expected/mbtiles/copy_diff.txt b/tests/expected/mbtiles/copy_diff.txt index 9d29df970..c621789b1 100644 --- a/tests/expected/mbtiles/copy_diff.txt +++ b/tests/expected/mbtiles/copy_diff.txt @@ -1,2 +1,2 @@ [INFO ] Comparing ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) and ./tests/fixtures/mbtiles/world_cities_modified.mbtiles (flat) into a new file tests/mbtiles_temp_files/world_cities_diff.mbtiles (flat) -[INFO ] Adding a new metadata value agg_tiles_hash = 958D6E4F59AC4B5334076BADBC505CAB in tests/mbtiles_temp_files/world_cities_diff.mbtiles +[INFO ] Adding a new metadata value agg_tiles_hash = C9B90B490DF381D93F8CD8102A469F47 in tests/mbtiles_temp_files/world_cities_diff.mbtiles diff --git a/tests/fixtures/mbtiles/world_cities_diff.mbtiles b/tests/fixtures/mbtiles/world_cities_diff.mbtiles index fda85a3aaa003aa32f42d9e598e2c061c7c3199a..58e0057cedec4fe09fc2f6a6135daa17419c5c96 100644 GIT binary patch delta 99 zcmZorXi%6S&B#4b#+j9yK`-2EW6C^co+k{Zw8?P`hWMWX-Q}Baj@@7^;A)u%zL<&Z3{>IA11pr%+6UqPp delta 67 zcmV-J0KETzAb=o{8vzQD976>P04{F0v1FbD4DtX20k!}E`2n^8OAF;T{uRf0h;+-qaXqWO$7C$le5VVt?KF_%kOq0yCOAK2gS*Rh&VuXu-ymMe`ZgPG(=A!nk&0V