From 18125b9ec72a1607a26261048be1ae6fb68f4cdb Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 24 Jun 2024 19:06:34 -0400 Subject: [PATCH] wip --- mbtiles/tests/copy.rs | 286 +++++++++--------- .../snapshots/copy__databases@hash__bdr.snap | 75 +++++ 2 files changed, 221 insertions(+), 140 deletions(-) create mode 100644 mbtiles/tests/snapshots/copy__databases@hash__bdr.snap diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 721fdf1a7..b42b00b4d 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -20,6 +20,7 @@ use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; use serde::Serialize; use sqlx::{query, query_as, Executor as _, Row, SqliteConnection}; +use tokio::runtime::Handle; const TILES_V1: &str = " INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES @@ -219,146 +220,151 @@ impl Databases { /// These dbs will be used by other tests to check against in various conditions. #[fixture] #[once] +#[allow(clippy::too_many_lines)] fn databases() -> Databases { - futures::executor::block_on(async { - let mut result = Databases::default(); - for &mbt_typ in &[Flat, FlatWithHash, Normalized] { - let typ = shorten(mbt_typ); - - // ----------------- empty_no_hash ----------------- - let (raw_empty_mbt, mut raw_empty_cn) = - new_file_no_hash!(databases, mbt_typ, "", "", "{typ}__empty-no-hash"); - let dmp = dump(&mut raw_empty_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__empty-no-hash"); - result.add( - "empty_no_hash", - mbt_typ, - dmp, - raw_empty_mbt, - None, - raw_empty_cn, - ); - - // ----------------- empty ----------------- - let (empty_mbt, mut empty_cn) = open!(databases, "{typ}__empty"); - copy!(result.path("empty_no_hash", mbt_typ), path(&empty_mbt)); - let dmp = dump(&mut empty_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__empty"); - let hash = empty_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); + tokio::task::block_in_place(|| { + Handle::current().block_on(async { + let mut result = Databases::default(); + for &mbt_typ in &[Flat, FlatWithHash, Normalized] { + let typ = shorten(mbt_typ); + + // ----------------- empty_no_hash ----------------- + let (raw_empty_mbt, mut raw_empty_cn) = + new_file_no_hash!(databases, mbt_typ, "", "", "{typ}__empty-no-hash"); + let dmp = dump(&mut raw_empty_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__empty-no-hash"); + result.add( + "empty_no_hash", + mbt_typ, + dmp, + raw_empty_mbt, + None, + raw_empty_cn, + ); + + // ----------------- empty ----------------- + let (empty_mbt, mut empty_cn) = open!(databases, "{typ}__empty"); + copy!(result.path("empty_no_hash", mbt_typ), path(&empty_mbt)); + let dmp = dump(&mut empty_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__empty"); + let hash = empty_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); + } + result.add("empty", mbt_typ, dmp, empty_mbt, Some(hash), empty_cn); + + // ----------------- v1_no_hash ----------------- + let (raw_mbt, mut raw_cn) = new_file_no_hash!( + databases, + mbt_typ, + METADATA_V1, + TILES_V1, + "{typ}__v1-no-hash" + ); + let dmp = dump(&mut raw_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__v1-no-hash"); + result.add("v1_no_hash", mbt_typ, dmp, raw_mbt, None, raw_cn); + + // ----------------- v1 ----------------- + let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1"); + copy!(result.path("v1_no_hash", mbt_typ), path(&v1_mbt)); + let dmp = dump(&mut v1_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__v1"); + let hash = v1_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); + } + result.add("v1", mbt_typ, dmp, v1_mbt, Some(hash), v1_cn); + + // ----------------- v2 ----------------- + let (v2_mbt, mut v2_cn) = + new_file!(databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2"); + let dmp = dump(&mut v2_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__v2"); + let hash = v2_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"3BCDEE3F52407FF1315629298CB99133"); + } + result.add("v2", mbt_typ, dmp, v2_mbt, Some(hash), v2_cn); + + // ----------------- dif (v1 -> v2) ----------------- + let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif"); + copy! { + result.path("v1", mbt_typ), + path(&dif_mbt), + diff_with_file => Some((result.path("v2", mbt_typ), Whole)), + }; + let dmp = dump(&mut dif_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__dif"); + let hash = dif_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1"); + } + result.add("dif", mbt_typ, dmp, dif_mbt, Some(hash), dif_cn); + + // ----------------- bin-diff-raw bdr (v1 -> v2) ----------------- + if mbt_typ == FlatWithHash { + let (bdr_mbt, mut bdr_cn) = open!(databases, "{typ}__bdr"); + eprintln!("TEST: bin-diff-raw (v1 -> v2) for {typ}..."); + eprintln!("INFO: bdr_mbt={bdr_mbt}"); + copy! { + result.path("v1", mbt_typ), + path(&bdr_mbt), + diff_with_file => Some((result.path("v2", mbt_typ), BinDiffRaw)), + }; + let dmp = dump(&mut bdr_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__bdr"); + let hash = bdr_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"585A88FEEC740448FF1EB4F96088FFE3"); + } + result.add("bdr", mbt_typ, dmp, bdr_mbt, Some(hash), bdr_cn); + } + + // ----------------- v1_clone ----------------- + let (v1_clone_mbt, v1_clone_cn) = open!(databases, "{typ}__v1-clone"); + let dmp = copy_dump!(result.path("v1", mbt_typ), path(&v1_clone_mbt)); + let hash = v1_clone_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); + } + result.add( + "v1_clone", + mbt_typ, + dmp, + v1_clone_mbt, + Some(hash), + v1_clone_cn, + ); + + // ----------------- dif_empty (v1 -> v1_clone) ----------------- + let (dif_empty_mbt, mut dif_empty_cn) = open!(databases, "{typ}__dif_empty"); + copy! { + result.path("v1", mbt_typ), + path(&dif_empty_mbt), + diff_with_file => Some((result.path("v1_clone", mbt_typ), Whole)), + }; + let dmp = dump(&mut dif_empty_cn).await.unwrap(); + assert_dump!(&dmp, "{typ}__dif_empty"); + let hash = dif_empty_mbt.open_and_validate(Off, Verify).await.unwrap(); + allow_duplicates! { + assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); + } + result.add( + "dif_empty", + mbt_typ, + dmp, + dif_empty_mbt, + Some(hash), + dif_empty_cn, + ); } - result.add("empty", mbt_typ, dmp, empty_mbt, Some(hash), empty_cn); - - // ----------------- v1_no_hash ----------------- - let (raw_mbt, mut raw_cn) = new_file_no_hash!( - databases, - mbt_typ, - METADATA_V1, - TILES_V1, - "{typ}__v1-no-hash" - ); - let dmp = dump(&mut raw_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__v1-no-hash"); - result.add("v1_no_hash", mbt_typ, dmp, raw_mbt, None, raw_cn); - - // ----------------- v1 ----------------- - let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1"); - copy!(result.path("v1_no_hash", mbt_typ), path(&v1_mbt)); - let dmp = dump(&mut v1_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__v1"); - let hash = v1_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); - } - result.add("v1", mbt_typ, dmp, v1_mbt, Some(hash), v1_cn); - - // ----------------- v2 ----------------- - let (v2_mbt, mut v2_cn) = - new_file!(databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2"); - let dmp = dump(&mut v2_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__v2"); - let hash = v2_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"3BCDEE3F52407FF1315629298CB99133"); - } - result.add("v2", mbt_typ, dmp, v2_mbt, Some(hash), v2_cn); - - // ----------------- dif (v1 -> v2) ----------------- - let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif"); - copy! { - result.path("v1", mbt_typ), - path(&dif_mbt), - diff_with_file => Some((result.path("v2", mbt_typ), Whole)), - }; - let dmp = dump(&mut dif_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__dif"); - let hash = dif_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1"); - } - result.add("dif", mbt_typ, dmp, dif_mbt, Some(hash), dif_cn); - - // ----------------- dif_bdr (v1 -> v2) ----------------- - // if mbt_typ == FlatWithHash { - // let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif_bdr"); - // copy! { - // result.path("v1", mbt_typ), - // path(&dif_mbt), - // diff_with_file => Some((result.path("v2", mbt_typ), BinDiffRaw)), - // }; - // let dmp = dump(&mut dif_cn).await.unwrap(); - // assert_dump!(&dmp, "{typ}__dif_bdr"); - // let hash = dif_mbt.open_and_validate(Off, Verify).await.unwrap(); - // allow_duplicates! { - // assert_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1"); - // } - // result.add("dif_bdr", mbt_typ, dmp, dif_mbt, Some(hash), dif_cn); - // } - - // ----------------- v1_clone ----------------- - let (v1_clone_mbt, v1_clone_cn) = open!(databases, "{typ}__v1-clone"); - let dmp = copy_dump!(result.path("v1", mbt_typ), path(&v1_clone_mbt)); - let hash = v1_clone_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); - } - result.add( - "v1_clone", - mbt_typ, - dmp, - v1_clone_mbt, - Some(hash), - v1_clone_cn, - ); - - // ----------------- dif_empty (v1 -> v1_clone) ----------------- - let (dif_empty_mbt, mut dif_empty_cn) = open!(databases, "{typ}__dif_empty"); - copy! { - result.path("v1", mbt_typ), - path(&dif_empty_mbt), - diff_with_file => Some((result.path("v1_clone", mbt_typ), Whole)), - }; - let dmp = dump(&mut dif_empty_cn).await.unwrap(); - assert_dump!(&dmp, "{typ}__dif_empty"); - let hash = dif_empty_mbt.open_and_validate(Off, Verify).await.unwrap(); - allow_duplicates! { - assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); - } - result.add( - "dif_empty", - mbt_typ, - dmp, - dif_empty_mbt, - Some(hash), - dif_empty_cn, - ); - } - result + result + }) }) } -#[actix_rt::test] +#[tokio::test] async fn update() -> MbtResult<()> { let (mbt, mut cn) = new_file_no_hash!(databases, Flat, METADATA_V1, TILES_V1, "update"); mbt.update_metadata(&mut cn, UpdateZoomType::Reset).await?; @@ -370,7 +376,7 @@ async fn update() -> MbtResult<()> { #[rstest] #[trace] -#[actix_rt::test] +#[tokio::test(flavor = "multi_thread")] async fn convert( #[values(Flat, FlatWithHash, Normalized)] frm_type: MbtTypeCli, #[values(Flat, FlatWithHash, Normalized)] dst_type: MbtTypeCli, @@ -462,7 +468,7 @@ async fn convert( #[rstest] #[trace] -#[actix_rt::test] +#[tokio::test(flavor = "multi_thread")] async fn diff_and_patch( #[values(Flat, FlatWithHash, Normalized)] a_type: MbtTypeCli, #[values(Flat, FlatWithHash, Normalized)] b_type: MbtTypeCli, @@ -523,7 +529,7 @@ async fn diff_and_patch( #[rstest] #[trace] -#[actix_rt::test] +#[tokio::test(flavor = "multi_thread")] #[ignore] async fn diff_and_patch_bsdiff( #[values(FlatWithHash)] a_type: MbtTypeCli, @@ -586,7 +592,7 @@ async fn diff_and_patch_bsdiff( #[rstest] #[trace] -#[actix_rt::test] +#[tokio::test(flavor = "multi_thread")] async fn patch_on_copy( #[values(Flat, FlatWithHash, Normalized)] v1_type: MbtTypeCli, #[values(Flat, FlatWithHash, Normalized)] dif_type: MbtTypeCli, @@ -612,7 +618,7 @@ async fn patch_on_copy( } /// A simple tester to run specific values -#[actix_rt::test] +#[tokio::test(flavor = "multi_thread")] #[ignore] async fn test_one() { // This will cause an error if ran together with other tests diff --git a/mbtiles/tests/snapshots/copy__databases@hash__bdr.snap b/mbtiles/tests/snapshots/copy__databases@hash__bdr.snap new file mode 100644 index 000000000..47410135c --- /dev/null +++ b/mbtiles/tests/snapshots/copy__databases@hash__bdr.snap @@ -0,0 +1,75 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'bsdiffraw' +sql = ''' +CREATE TABLE bsdiffraw ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + patch_data blob NOT NULL, + uncompressed_tile_xxh3_64 integer NOT NULL, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(1F8B08000000000002FF636380004628CD0AA11A20941100535E417C1F000000), 479130493 )', + '( 5, 1, 2, blob(1F8B08000000000002FF636080004E065490975FA29B9A5B505209001CF9666321000000), -1097843426 )', + '( 5, 1, 3, blob(1F8B08000000000002FF03000000000000000000), 953390274 )', + '( 6, 1, 4, blob(1F8B08000000000002FF636380002628CD0AA11A20945122007B56B74220000000), 386481748 )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "585A88FEEC740448FF1EB4F96088FFE3" )', + '( "agg_tiles_hash_after_apply", "3BCDEE3F52407FF1315629298CB99133" )', + '( "agg_tiles_hash_before_apply", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v2" )', + '( "md-new", "value - new" )', + '( "md-remove", NULL )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 2, 2, NULL, "" )', + '( 5, 2, 3, NULL, "" )', + '( 5, 3, 7, blob(new), "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( 5, 3, 8, blob(new), "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( 6, 2, 6, NULL, "" )', +] + +[[]] +type = 'index' +tbl_name = 'bsdiffraw' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash'''