diff --git a/Cargo.lock b/Cargo.lock index a09f224f3..0fadd1617 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1799,6 +1799,7 @@ dependencies = [ "actix-rt", "anyhow", "clap", + "env_logger", "futures", "log", "martin-tile-utils", diff --git a/justfile b/justfile index 503a4df79..a8aacc642 100644 --- a/justfile +++ b/justfile @@ -110,6 +110,7 @@ test-int: clean-test install-sqlx # Run integration tests and save its output as the new expected output bless: start clean-test + rm -rf tests/temp cargo test --features bless-tests tests/test.sh rm -rf tests/expected @@ -216,7 +217,7 @@ git-pre-push: stop start # Update sqlite database schema. prepare-sqlite: install-sqlx mkdir -p martin-mbtiles/.sqlx - cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests + cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/mbtiles/world_cities.mbtiles -- --lib --tests # Install SQLX cli if not already installed. [private] diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index 3392c25a9..fd746f043 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -12,7 +12,7 @@ rust-version.workspace = true [features] # TODO: Disable "cli" feature in default builds default = ["cli", "native-tls"] -cli = ["dep:anyhow", "dep:clap", "dep:tokio"] +cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:tokio"] # One of the following two must be used native-tls = ["sqlx/runtime-tokio-native-tls"] rustls = ["sqlx/runtime-tokio-rustls"] @@ -30,6 +30,7 @@ tilejson.workspace = true # Bin dependencies anyhow = { workspace = true, optional = true } clap = { workspace = true, optional = true } +env_logger = { workspace = true, optional = true } serde_yaml.workspace = true sqlite-hashes.workspace = true tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 5a7b0f542..4df966ce1 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -1,11 +1,10 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; +use log::{error, LevelFilter}; use martin_mbtiles::{ apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions, }; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::{Connection, SqliteConnection}; #[derive(Parser, PartialEq, Eq, Debug)] #[command( @@ -73,9 +72,23 @@ enum Commands { } #[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); +async fn main() { + env_logger::builder() + .filter_level(LevelFilter::Info) + .format_indent(None) + .format_module_path(false) + .format_target(false) + .format_timestamp(None) + .init(); + if let Err(err) = main_int().await { + error!("{err}"); + std::process::exit(1); + } +} + +async fn main_int() -> anyhow::Result<()> { + let args = Args::parse(); match args.command { Commands::MetaAll { file } => { meta_print_all(file.as_path()).await?; @@ -109,8 +122,7 @@ async fn main() -> anyhow::Result<()> { async fn meta_print_all(file: &Path) -> anyhow::Result<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(true).await?; let metadata = mbt.get_metadata(&mut conn).await?; println!("{}", serde_yaml::to_string(&metadata)?); Ok(()) @@ -118,8 +130,7 @@ async fn meta_print_all(file: &Path) -> anyhow::Result<()> { async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(true).await?; if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? { println!("{s}"); } @@ -128,8 +139,7 @@ async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> { async fn meta_set_value(file: &Path, key: &str, value: Option) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(false).await?; mbt.set_metadata_value(&mut conn, key, value).await } @@ -139,8 +149,7 @@ async fn validate_mbtiles( update_agg_tiles_hash: bool, ) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(!update_agg_tiles_hash).await?; mbt.check_integrity(&mut conn, check_type).await?; mbt.check_each_tile_hash(&mut conn).await?; if update_agg_tiles_hash { diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index 6a1c37242..a3d350dc9 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -435,7 +435,9 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + let filepath = self.filepath(); if integrity_check == IntegrityCheckType::Off { + info!("Skipping integrity check for {filepath}"); return Ok(()); } @@ -452,13 +454,14 @@ impl Mbtiles { if result.len() > 1 || result.get(0).ok_or(FailedIntegrityCheck( - self.filepath().to_string(), + filepath.to_string(), vec!["SQLite could not perform integrity check".to_string()], ))? != "ok" { return Err(FailedIntegrityCheck(self.filepath().to_string(), result)); } + info!("{integrity_check:?} integrity check passed for {filepath}"); Ok(()) } @@ -466,16 +469,17 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + let filepath = self.filepath(); let Some(stored) = self.get_agg_tiles_hash(&mut *conn).await? else { - return Err(AggHashValueNotFound(self.filepath().to_string())); + return Err(AggHashValueNotFound(filepath.to_string())); }; - let computed = calc_agg_tiles_hash(&mut *conn).await?; if stored != computed { - let file = self.filepath().to_string(); + let file = filepath.to_string(); return Err(AggHashMismatch(computed, stored, file)); } + info!("The agg_tiles_hashes={computed} has been verified for {filepath}"); Ok(()) } @@ -486,23 +490,15 @@ impl Mbtiles { { let old_hash = self.get_agg_tiles_hash(&mut *conn).await?; let hash = calc_agg_tiles_hash(&mut *conn).await?; + let path = self.filepath(); if old_hash.as_ref() == Some(&hash) { - info!( - "agg_tiles_hash is already set to the correct value `{hash}` in {}", - self.filepath() - ); + info!("agg_tiles_hash is already set to the correct value `{hash}` in {path}"); Ok(()) } else { if let Some(old_hash) = old_hash { - info!( - "Updating agg_tiles_hash from {old_hash} to {hash} in {}", - self.filepath() - ); + info!("Updating agg_tiles_hash from {old_hash} to {hash} in {path}"); } else { - info!( - "Initializing agg_tiles_hash to {hash} in {}", - self.filepath() - ); + info!("Creating new metadata value agg_tiles_hash = {hash} in {path}"); } self.set_metadata_value(&mut *conn, "agg_tiles_hash", Some(hash)) .await @@ -550,7 +546,10 @@ impl Mbtiles { v.get(0), v.get(1), )) - }) + })?; + + info!("All tile hashes are valid for {}", self.filepath()); + Ok(()) } } @@ -610,7 +609,7 @@ mod tests { #[actix_rt::test] async fn mbtiles_meta() { - let filepath = "../tests/fixtures/files/geography-class-jpg.mbtiles"; + let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"; let mbt = Mbtiles::new(filepath).unwrap(); assert_eq!(mbt.filepath(), filepath); assert_eq!(mbt.filename(), "geography-class-jpg"); @@ -618,7 +617,7 @@ mod tests { #[actix_rt::test] async fn metadata_jpeg() { - let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; let metadata = mbt.get_metadata(&mut conn).await.unwrap(); let tj = metadata.tilejson; @@ -635,7 +634,7 @@ mod tests { #[actix_rt::test] async fn metadata_mvt() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let metadata = mbt.get_metadata(&mut conn).await.unwrap(); let tj = metadata.tilejson; @@ -666,7 +665,7 @@ mod tests { #[actix_rt::test] async fn metadata_get_key() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap(); assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706"); @@ -726,15 +725,15 @@ mod tests { #[actix_rt::test] async fn detect_type() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::Flat); - let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::FlatWithHash); - let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::Normalized); @@ -745,7 +744,7 @@ mod tests { #[actix_rt::test] async fn validate_valid_file() { - let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; mbt.check_integrity(&mut conn, IntegrityCheckType::Quick) .await @@ -755,7 +754,7 @@ mod tests { #[actix_rt::test] async fn validate_invalid_file() { let (mut conn, mbt) = - open("../tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles").await; + open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await; let result = mbt.check_agg_tiles_hashes(&mut conn).await; assert!(matches!(result, Err(MbtError::AggHashMismatch(..)))); } diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index 6827f410f..8e8ced1b5 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -603,14 +603,14 @@ mod tests { #[actix_rt::test] async fn copy_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_flat_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, Flat).await } #[actix_rt::test] async fn copy_flat_from_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_flat_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); @@ -619,7 +619,7 @@ mod tests { #[actix_rt::test] async fn copy_flat_from_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_flat_from_normalized_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, Some(Flat), Flat).await @@ -627,14 +627,14 @@ mod tests { #[actix_rt::test] async fn copy_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from("file:copy_flat_with_hash_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, FlatWithHash).await } #[actix_rt::test] async fn copy_flat_with_hash_from_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_flat_with_hash_from_flat_tables_mem_db?mode=memory&cache=shared", ); @@ -643,7 +643,7 @@ mod tests { #[actix_rt::test] async fn copy_flat_with_hash_from_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from( "file:copy_flat_with_hash_from_normalized_tables_mem_db?mode=memory&cache=shared", ); @@ -652,14 +652,14 @@ mod tests { #[actix_rt::test] async fn copy_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_normalized_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, Normalized).await } #[actix_rt::test] async fn copy_normalized_from_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_normalized_from_flat_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, Some(Normalized), Normalized).await @@ -667,7 +667,7 @@ mod tests { #[actix_rt::test] async fn copy_normalized_from_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_normalized_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); @@ -676,7 +676,7 @@ mod tests { #[actix_rt::test] async fn copy_with_min_max_zoom() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) @@ -686,7 +686,7 @@ mod tests { #[actix_rt::test] async fn copy_with_zoom_levels() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) @@ -697,11 +697,11 @@ mod tests { #[actix_rt::test] async fn copy_with_diff_with_file() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let dst = PathBuf::from("file:copy_with_diff_with_file_mem_db?mode=memory&cache=shared"); let diff_file = - PathBuf::from("../tests/fixtures/files/geography-class-jpg-modified.mbtiles"); + PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); @@ -744,10 +744,10 @@ mod tests { #[actix_rt::test] async fn ignore_dst_type_when_copy_to_existing() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from( "file:ignore_dst_type_when_copy_to_existing_mem_db?mode=memory&cache=shared", ); @@ -761,8 +761,8 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_abort_mode() { - let src = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); - let dst = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); + let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort); @@ -775,10 +775,10 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_override_mode() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared"); @@ -804,10 +804,10 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_ignore_mode() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared"); @@ -858,7 +858,7 @@ mod tests { #[actix_rt::test] async fn apply_flat_diff_file() -> MbtResult<()> { // Copy the src file to an in-memory DB - let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) @@ -866,11 +866,11 @@ mod tests { .await?; // Apply diff to the src data in in-memory DB - let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles"); + let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/files/world_cities_modified.mbtiles"; + let path = "../tests/fixtures/mbtiles/world_cities_modified.mbtiles"; attach_other_db(&mut src_conn, path).await?; assert!( @@ -886,7 +886,7 @@ mod tests { #[actix_rt::test] async fn apply_normalized_diff_file() -> MbtResult<()> { // Copy the src file to an in-memory DB - let src_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) @@ -894,11 +894,11 @@ mod tests { .await?; // Apply diff to the src data in in-memory DB - let diff_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg-diff.mbtiles"); + let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/files/geography-class-jpg-modified.mbtiles"; + let path = "../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"; attach_other_db(&mut src_conn, path).await?; assert!( diff --git a/tests/config.yaml b/tests/config.yaml index 1f9dc1c66..471f31428 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -162,7 +162,7 @@ postgres: pmtiles: sources: - pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles sprites: paths: tests/fixtures/sprites/src1 diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index c51f1f02f..e8229fd9a 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -133,23 +133,27 @@ postgres: schema: public function: function_zxy_row_key pmtiles: - paths: tests/fixtures/files + paths: + - tests/fixtures/mbtiles + - tests/fixtures/pmtiles sources: - png: tests/fixtures/files/png.pmtiles - stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - webp2: tests/fixtures/files/webp2.pmtiles + png: tests/fixtures/pmtiles/png.pmtiles + stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + webp2: tests/fixtures/pmtiles/webp2.pmtiles mbtiles: - paths: tests/fixtures/files + paths: + - tests/fixtures/mbtiles + - tests/fixtures/pmtiles sources: - geography-class-jpg: tests/fixtures/files/geography-class-jpg.mbtiles - geography-class-jpg-diff: tests/fixtures/files/geography-class-jpg-diff.mbtiles - geography-class-jpg-modified: tests/fixtures/files/geography-class-jpg-modified.mbtiles - geography-class-png: tests/fixtures/files/geography-class-png.mbtiles - geography-class-png-no-bounds: tests/fixtures/files/geography-class-png-no-bounds.mbtiles - json: tests/fixtures/files/json.mbtiles - uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles - webp: tests/fixtures/files/webp.mbtiles - world_cities: tests/fixtures/files/world_cities.mbtiles - world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles - world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles - zoomed_world_cities: tests/fixtures/files/zoomed_world_cities.mbtiles + geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles + geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles + geography-class-jpg-modified: tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles + geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles + geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles + json: tests/fixtures/mbtiles/json.mbtiles + uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles + webp: tests/fixtures/mbtiles/webp.mbtiles + world_cities: tests/fixtures/mbtiles/world_cities.mbtiles + world_cities_diff: tests/fixtures/mbtiles/world_cities_diff.mbtiles + world_cities_modified: tests/fixtures/mbtiles/world_cities_modified.mbtiles + zoomed_world_cities: tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/expected/given_config.yaml b/tests/expected/given_config.yaml index 033c66283..6ba51282e 100644 --- a/tests/expected/given_config.yaml +++ b/tests/expected/given_config.yaml @@ -159,7 +159,7 @@ postgres: - 90.0 pmtiles: sources: - pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles sprites: paths: tests/fixtures/sprites/src1 sources: diff --git a/tests/expected/mbtiles/copy_diff.txt b/tests/expected/mbtiles/copy_diff.txt index e69de29bb..4fc7c848d 100644 --- a/tests/expected/mbtiles/copy_diff.txt +++ b/tests/expected/mbtiles/copy_diff.txt @@ -0,0 +1 @@ +[INFO ] Creating new metadata value agg_tiles_hash = C7E2E5A9BA04693994DB1F57D1DF5646 in tests/temp/world_cities_diff.mbtiles diff --git a/tests/expected/mbtiles/copy_diff2.txt b/tests/expected/mbtiles/copy_diff2.txt index e69de29bb..cddb2637d 100644 --- a/tests/expected/mbtiles/copy_diff2.txt +++ b/tests/expected/mbtiles/copy_diff2.txt @@ -0,0 +1 @@ +[INFO ] Creating new metadata value agg_tiles_hash = D41D8CD98F00B204E9800998ECF8427E in tests/temp/world_cities_diff_modified.mbtiles diff --git a/tests/expected/mbtiles/validate-bad.txt b/tests/expected/mbtiles/validate-bad.txt new file mode 100644 index 000000000..242a6aaaa --- /dev/null +++ b/tests/expected/mbtiles/validate-bad.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for ./tests/fixtures/files/bad_hash.mbtiles +[INFO ] All tile hashes are valid for ./tests/fixtures/files/bad_hash.mbtiles +[ERROR] Computed aggregate tiles hash D4E1030D57751A0B45A28A71267E46B8 does not match tile data in metadata CAFEC0DEDEADBEEFDEADBEEFDEADBEEF for MBTile file ./tests/fixtures/files/bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix.txt b/tests/expected/mbtiles/validate-fix.txt new file mode 100644 index 000000000..ded275f11 --- /dev/null +++ b/tests/expected/mbtiles/validate-fix.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles +[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles +[INFO ] Updating agg_tiles_hash from CAFEC0DEDEADBEEFDEADBEEFDEADBEEF to D4E1030D57751A0B45A28A71267E46B8 in tests/temp/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix2.txt b/tests/expected/mbtiles/validate-fix2.txt new file mode 100644 index 000000000..250ba58e6 --- /dev/null +++ b/tests/expected/mbtiles/validate-fix2.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles +[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles +[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for tests/temp/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-ok.txt b/tests/expected/mbtiles/validate-ok.txt new file mode 100644 index 000000000..87a15d314 --- /dev/null +++ b/tests/expected/mbtiles/validate-ok.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles +[INFO ] All tile hashes are valid for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles +[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/bad_hash.mbtiles b/tests/fixtures/files/bad_hash.mbtiles new file mode 100644 index 000000000..33943e699 Binary files /dev/null and b/tests/fixtures/files/bad_hash.mbtiles differ diff --git a/tests/fixtures/files/invalid/invalid-tile-format.mbtiles b/tests/fixtures/files/invalid-tile-format.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid-tile-format.mbtiles rename to tests/fixtures/files/invalid-tile-format.mbtiles diff --git a/tests/fixtures/files/invalid/invalid.mbtiles b/tests/fixtures/files/invalid.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid.mbtiles rename to tests/fixtures/files/invalid.mbtiles diff --git a/tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles b/tests/fixtures/files/invalid_zoomed_world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles rename to tests/fixtures/files/invalid_zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg-diff.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg-diff.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg-modified.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg-modified.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg.mbtiles diff --git a/tests/fixtures/files/geography-class-png-no-bounds.mbtiles b/tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-png-no-bounds.mbtiles rename to tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles diff --git a/tests/fixtures/files/geography-class-png.mbtiles b/tests/fixtures/mbtiles/geography-class-png.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-png.mbtiles rename to tests/fixtures/mbtiles/geography-class-png.mbtiles diff --git a/tests/fixtures/files/json.mbtiles b/tests/fixtures/mbtiles/json.mbtiles similarity index 100% rename from tests/fixtures/files/json.mbtiles rename to tests/fixtures/mbtiles/json.mbtiles diff --git a/tests/fixtures/files/uncompressed_mvt.mbtiles b/tests/fixtures/mbtiles/uncompressed_mvt.mbtiles similarity index 100% rename from tests/fixtures/files/uncompressed_mvt.mbtiles rename to tests/fixtures/mbtiles/uncompressed_mvt.mbtiles diff --git a/tests/fixtures/files/webp.mbtiles b/tests/fixtures/mbtiles/webp.mbtiles similarity index 100% rename from tests/fixtures/files/webp.mbtiles rename to tests/fixtures/mbtiles/webp.mbtiles diff --git a/tests/fixtures/files/world_cities.mbtiles b/tests/fixtures/mbtiles/world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities.mbtiles rename to tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/fixtures/files/world_cities_diff.mbtiles b/tests/fixtures/mbtiles/world_cities_diff.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities_diff.mbtiles rename to tests/fixtures/mbtiles/world_cities_diff.mbtiles diff --git a/tests/fixtures/files/world_cities_modified.mbtiles b/tests/fixtures/mbtiles/world_cities_modified.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities_modified.mbtiles rename to tests/fixtures/mbtiles/world_cities_modified.mbtiles diff --git a/tests/fixtures/files/zoomed_world_cities.mbtiles b/tests/fixtures/mbtiles/zoomed_world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/zoomed_world_cities.mbtiles rename to tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/png.pmtiles b/tests/fixtures/pmtiles/png.pmtiles similarity index 100% rename from tests/fixtures/files/png.pmtiles rename to tests/fixtures/pmtiles/png.pmtiles diff --git a/tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles b/tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles similarity index 100% rename from tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles rename to tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles diff --git a/tests/fixtures/files/webp2.pmtiles b/tests/fixtures/pmtiles/webp2.pmtiles similarity index 100% rename from tests/fixtures/files/webp2.pmtiles rename to tests/fixtures/pmtiles/webp2.pmtiles diff --git a/tests/mb_server_test.rs b/tests/mb_server_test.rs index 912c92a0f..94e1d1dff 100644 --- a/tests/mb_server_test.rs +++ b/tests/mb_server_test.rs @@ -34,10 +34,10 @@ fn test_get(path: &str) -> TestRequest { const CONFIG: &str = indoc! {" mbtiles: sources: - m_json: tests/fixtures/files/json.mbtiles - m_mvt: tests/fixtures/files/world_cities.mbtiles - m_raw_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles - m_webp: tests/fixtures/files/webp.mbtiles + m_json: tests/fixtures/mbtiles/json.mbtiles + m_mvt: tests/fixtures/mbtiles/world_cities.mbtiles + m_raw_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles + m_webp: tests/fixtures/mbtiles/webp.mbtiles "}; #[actix_rt::test] diff --git a/tests/pg_table_source_test.rs b/tests/pg_table_source_test.rs index 4207138d4..5bba0b85b 100644 --- a/tests/pg_table_source_test.rs +++ b/tests/pg_table_source_test.rs @@ -113,5 +113,5 @@ async fn table_source_schemas() { functions: false "}); let sources = mock_sources(cfg).await.0; - assert_eq!(sources.keys().collect::>(), vec!["MixPoints"],); + assert_eq!(sources.keys().collect::>(), vec!["MixPoints"]); } diff --git a/tests/pmt_server_test.rs b/tests/pmt_server_test.rs index bcb81da41..916601a87 100644 --- a/tests/pmt_server_test.rs +++ b/tests/pmt_server_test.rs @@ -34,12 +34,12 @@ fn test_get(path: &str) -> TestRequest { const CONFIG: &str = indoc! {" pmtiles: sources: - p_png: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + p_png: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles "}; #[actix_rt::test] async fn pmt_get_catalog() { - let path = "pmtiles: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles"; + let path = "pmtiles: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles"; let app = create_app! { path }; let req = test_get("/catalog").to_request(); diff --git a/tests/test.sh b/tests/test.sh index 846e1a3de..ab47e712a 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -150,7 +150,7 @@ echo "Test auto configured Martin" TEST_OUT_DIR="$(dirname "$0")/output/auto" mkdir -p "$TEST_OUT_DIR" -ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/files) +ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles) set -x $MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_1.txt & PROCESS_ID=`jobs -p` @@ -289,21 +289,34 @@ if [[ "$MBTILES_BIN" != "-" ]]; then $MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt" $MBTILES_BIN meta-all --help 2>&1 | tee "$TEST_OUT_DIR/meta-all_help.txt" - $MBTILES_BIN meta-all ./tests/fixtures/files/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt" + $MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt" $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt" - $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" - $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt" + $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" + $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt" + $MBTILES_BIN validate ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-ok.txt" + + set +e + $MBTILES_BIN validate ./tests/fixtures/files/bad_hash.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-bad.txt" + if [[ $? -eq 0 ]]; then + echo "ERROR: validate with bad_hash should have failed" + exit 1 + fi + set -e + + cp ./tests/fixtures/files/bad_hash.mbtiles "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" + $MBTILES_BIN validate --update-agg-tiles-hash "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix.txt" + $MBTILES_BIN validate "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix2.txt" # Create diff file $MBTILES_BIN copy \ - ./tests/fixtures/files/world_cities.mbtiles \ + ./tests/fixtures/mbtiles/world_cities.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff.mbtiles" \ - --diff-with-file ./tests/fixtures/files/world_cities_modified.mbtiles \ + --diff-with-file ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff.txt" if command -v sqlite3 > /dev/null; then # Apply this diff to the original version of the file - cp ./tests/fixtures/files/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles" + cp ./tests/fixtures/mbtiles/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles" sqlite3 "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \ -bail \ @@ -315,7 +328,7 @@ if [[ "$MBTILES_BIN" != "-" ]]; then # Ensure that applying the diff resulted in the modified version of the file $MBTILES_BIN copy \ --diff-with-file "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \ - ./tests/fixtures/files/world_cities_modified.mbtiles \ + ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff2.txt"