From 3ed75883d22f1835c0f3429dfba7cdef59e1036b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 9 Dec 2023 10:12:50 -0500 Subject: [PATCH] Add tiles table/view validation to `mbtiles validate` Make sure all values in the `tiles` table or view are correct: * zoom_level is between 0 and 30 (max allowed zoom) * the x,y values are within the limits for the corresponding zoom level * the column type of z,x,y are all integers * the `tile_data` is a NULL or a BLOB --- mbtiles/src/errors.rs | 3 + mbtiles/src/validation.rs | 63 +++++++++- mbtiles/tests/validate.rs | 119 ++++++++++++++++++ .../martin-cp/flat-with-hash_validate.txt | 1 + tests/expected/martin-cp/flat_validate.txt | 1 + .../martin-cp/normalized_validate.txt | 1 + tests/expected/mbtiles/validate-bad.txt | 1 + tests/expected/mbtiles/validate-fix.txt | 1 + tests/expected/mbtiles/validate-fix2.txt | 1 + tests/expected/mbtiles/validate-ok.txt | 1 + tests/fixtures/files/invalid-tile-idx.mbtiles | Bin 0 -> 4608 bytes .../fixtures/files/invalid-tile-type.mbtiles | Bin 0 -> 5632 bytes 12 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 mbtiles/tests/validate.rs create mode 100644 tests/fixtures/files/invalid-tile-idx.mbtiles create mode 100644 tests/fixtures/files/invalid-tile-type.mbtiles diff --git a/mbtiles/src/errors.rs b/mbtiles/src/errors.rs index 935237909..263fca43f 100644 --- a/mbtiles/src/errors.rs +++ b/mbtiles/src/errors.rs @@ -37,6 +37,9 @@ pub enum MbtError { #[error("At least one tile has mismatching hash: stored value is `{1}` != computed value `{2}` in MBTile file {0}")] IncorrectTileHash(String, String, String), + #[error("At least one tile has invalid tile index: zoom_level={1}, tile_column={2}, tile_row={3} in MBTile file {0}")] + InvalidTileIndex(String, String, String, String), + #[error("Computed aggregate tiles hash {0} does not match tile data in metadata {1} for MBTile file {2}")] AggHashMismatch(String, String, String), diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index e97cd49b6..d222c7df9 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -1,10 +1,11 @@ use std::collections::HashSet; +use std::str::from_utf8; #[cfg(feature = "cli")] use clap::ValueEnum; use enum_display::EnumDisplay; use log::{debug, info, warn}; -use martin_tile_utils::{Format, TileInfo}; +use martin_tile_utils::{Format, TileInfo, MAX_ZOOM}; use serde::Serialize; use serde_json::Value; use sqlx::sqlite::SqliteRow; @@ -18,6 +19,7 @@ use crate::queries::{ }; use crate::MbtError::{ AggHashMismatch, AggHashValueNotFound, FailedIntegrityCheck, IncorrectTileHash, + InvalidTileIndex, }; use crate::{invert_y_value, Mbtiles}; @@ -83,6 +85,7 @@ impl Mbtiles { self.open_readonly().await? }; self.check_integrity(&mut conn, check_type).await?; + self.check_tiles_type_validity(&mut conn).await?; self.check_each_tile_hash(&mut conn).await?; match agg_hash { AggHashType::Verify => self.check_agg_tiles_hashes(&mut conn).await, @@ -311,6 +314,64 @@ impl Mbtiles { Ok(()) } + /// Check that the tiles table has the expected column, row, zoom, and data values + pub async fn check_tiles_type_validity(&self, conn: &mut T) -> MbtResult<()> + where + for<'e> &'e mut T: SqliteExecutor<'e>, + { + let sql = format!( + " +SELECT zoom_level, tile_column, tile_row +FROM tiles +WHERE FALSE + OR typeof(zoom_level) != 'integer' + OR zoom_level < 0 + OR zoom_level > {MAX_ZOOM} + OR typeof(tile_column) != 'integer' + OR tile_column < 0 + OR tile_column >= (1 << zoom_level) + OR typeof(tile_row) != 'integer' + OR tile_row < 0 + OR tile_row >= (1 << zoom_level) + OR (typeof(tile_data) != 'blob' AND typeof(tile_data) != 'null') +LIMIT 1;" + ); + + if let Some(row) = query(&sql).fetch_optional(&mut *conn).await? { + let mut res: Vec = Vec::with_capacity(3); + for idx in (0..3).rev() { + use sqlx::ValueRef as _; + let raw = row.try_get_raw(idx)?; + if raw.is_null() { + res.push("NULL".to_string()); + } else if let Ok(v) = row.try_get::(idx) { + res.push(format!(r#""{v}" (TEXT)"#)); + } else if let Ok(v) = row.try_get::, _>(idx) { + res.push(format!( + r#""{}" (BLOB)"#, + from_utf8(&v).unwrap_or("") + )); + } else if let Ok(v) = row.try_get::(idx) { + res.push(format!("{v}")); + } else if let Ok(v) = row.try_get::(idx) { + res.push(format!(r#"{v} (REAL)"#)); + } else { + res.push(format!("{:?}", raw.type_info())); + } + } + + return Err(InvalidTileIndex( + self.filepath().to_string(), + res.pop().unwrap(), + res.pop().unwrap(), + res.pop().unwrap(), + )); + } + + info!("All values in the `tiles` table/view are valid for {self}"); + Ok(()) + } + pub async fn check_agg_tiles_hashes(&self, conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, diff --git a/mbtiles/tests/validate.rs b/mbtiles/tests/validate.rs new file mode 100644 index 000000000..7192d4538 --- /dev/null +++ b/mbtiles/tests/validate.rs @@ -0,0 +1,119 @@ +use martin_tile_utils::MAX_ZOOM; +use mbtiles::MbtError::InvalidTileIndex; +use mbtiles::{create_metadata_table, Mbtiles}; +use rstest::rstest; +use sqlx::{query, Executor as _, SqliteConnection}; + +#[ctor::ctor] +fn init() { + let _ = env_logger::builder().is_test(true).try_init(); +} + +async fn new(values: &str) -> (Mbtiles, SqliteConnection) { + let mbtiles = Mbtiles::new(":memory:").unwrap(); + let mut conn = mbtiles.open().await.unwrap(); + create_metadata_table(&mut conn).await.unwrap(); + + conn.execute( + "CREATE TABLE tiles ( + zoom_level integer, + tile_column integer, + tile_row integer, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + ) + .await + .unwrap(); + + let sql = format!( + "INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) + VALUES ({values});" + ); + query(&sql).execute(&mut conn).await.expect(&sql); + + (mbtiles, conn) +} + +macro_rules! ok { + ($($vals:tt)*) => {{ + let vals = format!($($vals)*); + let (mbt, mut conn) = new(&vals).await; + let res = mbt.check_tiles_type_validity(&mut conn).await; + assert!(res.is_ok(), "check_tiles_xyz_validity({vals}) = {res:?}, expected Ok"); + }}; +} + +macro_rules! err { + ($($vals:tt)*) => { + let vals = format!($($vals)*); + let (mbt, mut conn) = new(&vals).await; + match mbt.check_tiles_type_validity(&mut conn).await { + Ok(()) => panic!("check_tiles_xyz_validity({vals}) was expected to fail"), + Err(e) => match e { + InvalidTileIndex(..) => {} + _ => panic!("check_tiles_xyz_validity({vals}) = Err({e:?}), expected Err(InvalidTileIndex)"), + }, + } + }; +} + +#[rstest] +#[case("", ", 0, 0, NULL")] // test tile_zoom +#[case("0, ", ", 0, NULL")] // test tile_column +#[case("0, 0, ", ", NULL")] // test tile_row +#[trace] +#[actix_rt::test] +async fn integers(#[case] prefix: &str, #[case] suffix: &str) { + ok!("{prefix} 0 {suffix}"); + + err!("{prefix} NULL {suffix}"); + err!("{prefix} -1 {suffix}"); + err!("{prefix} 0.2 {suffix}"); + err!("{prefix} '' {suffix}"); + err!("{prefix} 'a' {suffix}"); + + err!("{prefix} CAST(1 AS BLOB) {suffix}"); + err!("{prefix} CAST('1' AS BLOB) {suffix}"); + + // These fail for some reason, probably due to internal SQLite casting/affinity rules? + // err!("{prefix} '1' {suffix}"); + // err!("{prefix} CAST(1 AS REAL) {suffix}"); + // err!("{prefix} CAST(1.0 AS NUMERIC) {suffix}"); + // err!("{prefix} CAST(1 AS TEXT) {suffix}"); +} + +#[rstest] +#[case("", ", 0, NULL")] // test tile_column +#[case("0, ", ", NULL")] // test tile_row +#[trace] +#[actix_rt::test] +async fn tile_coordinate(#[case] prefix: &str, #[case] suffix: &str) { + ok!("0, {prefix} 0 {suffix}"); + ok!("1, {prefix} 1 {suffix}"); + ok!("2, {prefix} 3 {suffix}"); + ok!("3, {prefix} 7 {suffix}"); + ok!("30, {prefix} 0 {suffix}"); + ok!("30, {prefix} 1073741823 {suffix}"); + + err!("0, {prefix} 1 {suffix}"); + err!("1, {prefix} 2 {suffix}"); + err!("2, {prefix} 4 {suffix}"); + err!("3, {prefix} 8 {suffix}"); + err!("30, {prefix} 1073741824 {suffix}"); + err!("{MAX_ZOOM}, {prefix} 1073741824 {suffix}"); + err!("{}, {prefix} 0 {suffix}", MAX_ZOOM + 1); // unsupported zoom +} + +#[actix_rt::test] +async fn tile_data() { + ok!("0, 0, 0, NULL"); + ok!("0, 0, 0, CAST('' AS BLOB)"); + ok!("0, 0, 0, CAST('abc' AS BLOB)"); + ok!("0, 0, 0, CAST(123 AS BLOB)"); + + err!("0, 0, 0, 0"); + err!("0, 0, 0, 0.1"); + err!("0, 0, 0, CAST('' AS TEXT)"); + err!("0, 0, 0, CAST('abc' AS TEXT)"); + err!("0, 0, 0, CAST(123 AS TEXT)"); +} diff --git a/tests/expected/martin-cp/flat-with-hash_validate.txt b/tests/expected/martin-cp/flat-with-hash_validate.txt index 48bfc7abf..0a344a967 100644 --- a/tests/expected/martin-cp/flat-with-hash_validate.txt +++ b/tests/expected/martin-cp/flat-with-hash_validate.txt @@ -1,2 +1,3 @@ [INFO ] Quick integrity check passed for tests/mbtiles_temp_files/cp_flat-with-hash.mbtiles +[INFO ] All values in the `tiles` table/view are valid for tests/mbtiles_temp_files/cp_flat-with-hash.mbtiles [INFO ] All tile hashes are valid for tests/mbtiles_temp_files/cp_flat-with-hash.mbtiles diff --git a/tests/expected/martin-cp/flat_validate.txt b/tests/expected/martin-cp/flat_validate.txt index d64ffd90d..19fea05ff 100644 --- a/tests/expected/martin-cp/flat_validate.txt +++ b/tests/expected/martin-cp/flat_validate.txt @@ -1,2 +1,3 @@ [INFO ] Quick integrity check passed for tests/mbtiles_temp_files/cp_flat.mbtiles +[INFO ] All values in the `tiles` table/view are valid for tests/mbtiles_temp_files/cp_flat.mbtiles [INFO ] Skipping per-tile hash validation because this is a flat MBTiles file diff --git a/tests/expected/martin-cp/normalized_validate.txt b/tests/expected/martin-cp/normalized_validate.txt index 3ef717bf5..ae3365050 100644 --- a/tests/expected/martin-cp/normalized_validate.txt +++ b/tests/expected/martin-cp/normalized_validate.txt @@ -1,2 +1,3 @@ [INFO ] Quick integrity check passed for tests/mbtiles_temp_files/cp_normalized.mbtiles +[INFO ] All values in the `tiles` table/view are valid for tests/mbtiles_temp_files/cp_normalized.mbtiles [INFO ] All tile hashes are valid for tests/mbtiles_temp_files/cp_normalized.mbtiles diff --git a/tests/expected/mbtiles/validate-bad.txt b/tests/expected/mbtiles/validate-bad.txt index 242a6aaaa..4ba1d3e25 100644 --- a/tests/expected/mbtiles/validate-bad.txt +++ b/tests/expected/mbtiles/validate-bad.txt @@ -1,3 +1,4 @@ [INFO ] Quick integrity check passed for ./tests/fixtures/files/bad_hash.mbtiles +[INFO ] All values in the `tiles` table/view are valid for ./tests/fixtures/files/bad_hash.mbtiles [INFO ] All tile hashes are valid for ./tests/fixtures/files/bad_hash.mbtiles [ERROR] Computed aggregate tiles hash D4E1030D57751A0B45A28A71267E46B8 does not match tile data in metadata CAFEC0DEDEADBEEFDEADBEEFDEADBEEF for MBTile file ./tests/fixtures/files/bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix.txt b/tests/expected/mbtiles/validate-fix.txt index 34ccfa33e..4e5815aae 100644 --- a/tests/expected/mbtiles/validate-fix.txt +++ b/tests/expected/mbtiles/validate-fix.txt @@ -1,3 +1,4 @@ [INFO ] Quick integrity check passed for tests/mbtiles_temp_files/fix_bad_hash.mbtiles +[INFO ] All values in the `tiles` table/view are valid for tests/mbtiles_temp_files/fix_bad_hash.mbtiles [INFO ] All tile hashes are valid for tests/mbtiles_temp_files/fix_bad_hash.mbtiles [INFO ] Updating agg_tiles_hash from CAFEC0DEDEADBEEFDEADBEEFDEADBEEF to D4E1030D57751A0B45A28A71267E46B8 in tests/mbtiles_temp_files/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix2.txt b/tests/expected/mbtiles/validate-fix2.txt index 291281294..23d4067bf 100644 --- a/tests/expected/mbtiles/validate-fix2.txt +++ b/tests/expected/mbtiles/validate-fix2.txt @@ -1,3 +1,4 @@ [INFO ] Quick integrity check passed for tests/mbtiles_temp_files/fix_bad_hash.mbtiles +[INFO ] All values in the `tiles` table/view are valid for tests/mbtiles_temp_files/fix_bad_hash.mbtiles [INFO ] All tile hashes are valid for tests/mbtiles_temp_files/fix_bad_hash.mbtiles [INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for tests/mbtiles_temp_files/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-ok.txt b/tests/expected/mbtiles/validate-ok.txt index 87a15d314..f7ad6204b 100644 --- a/tests/expected/mbtiles/validate-ok.txt +++ b/tests/expected/mbtiles/validate-ok.txt @@ -1,3 +1,4 @@ [INFO ] Quick integrity check passed for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles +[INFO ] All values in the `tiles` table/view are valid for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles [INFO ] All tile hashes are valid for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles [INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/invalid-tile-idx.mbtiles b/tests/fixtures/files/invalid-tile-idx.mbtiles new file mode 100644 index 0000000000000000000000000000000000000000..e672c480377eba878eb00dcdc66906365ead3927 GIT binary patch literal 4608 zcmeHKO=ufO6yDL=mVc6(G&RPAG}A4lFdYbo^7_jcvPHZcTBf1n;qp6|VR z^XAQ)H}js#mx{vUbV)i*=Fx-_QV3DbQmQCQKYV-OyVJS=bnFn&lU?{A*snwur#exX z{DCq(L3fb&A^)HCK$m(H7VJ|#Q^@z^Q}PjckF1eJ@(MXdROJU?{|8#^h&o(iwOYj! z7I!Ok=GG^Yg;+EZok(ResaP&Lo=oN9*<2b(9$bKBW9fx%b>M`zzRcwccVJR7tPWKT*|JS{ zG!{=p;F}tY>Z6HFBpb^{vnf56Nk%g1L^Ph!Q)7`tDxQg^Pj#t7V^wZ@+!@WJB4de6 zDjL-j*+?dv&5kAYbYG`>qQA-3-jTAI?(0B8*hobCR3Kn&S4cY^52>#%nB1*8V%Y;m ziCvcttqMcE1{n*4c$d-43qO@;hQw#iTA0l7!M zB43bC$Q|+lun*hc>VZD>1Zf|&3UNm;?z+S3Ko1@t4z`1$!^c6ibfO@Jc5kmbM1r{O z=~rJRyTT-dLqYg;Aq?U-{GvKc_THo;c>f8xrNHN~{ktA;Lt#S5*2|@?-m|A}b^FGI z)I0GnuWcLCkM)Pe(I4xxw~dF>k4g*Mzic0!c~o8=nR*06Gmi(iKfd;oVO*Ove!lzd z*XK_urN-$8DNvD0jHgrC$?RBeG8rGAh)s;;4Zol5NDyI{?h zI-eUa7U;e~dIpnR#o{a6q5|?`jXQMa;yj&MC>C{Jx>A)^t7#uJ>d4iDhA{h4!;(f@ z5!MRS<7=MYOZTF0*yI!mL&xF#LL`YcPM3D8e@RntQ{7n!dtXp7iO; zj><}G2dQ9p@WMck#e}9)ZC3)p>Ihlw`~zeZsUQOZl?G}_aLdGsYC}UyaZ1^9gk94% z=_b9ZYyQ!D%nM}v0xS=Oln0VAU3*%PZ3Gw<1eJ}ZYt^=3I;(3nF2QbR9R+4#OCF3f z50)}o9^a+$3C}O{HlE7hMBl`{dNJGLkQ-R7YraGM$y&}dg-de=GZ?IvYr0U$nXHM} zt8*Lt1;Apq=MT544ZtQa$62jzvH;mNW=V(pT)Wng4MbdAM;9S-EMX&%HR-DvZ*g0~ zqQb%R(DF<;((ji!otLX9n`aKIvS8GOtH|ss8juH*YG%tvqRMJGZGttJgFUyQ#6Zw8 zn1KEqxA_XrIxp=S#xI=);L|PFleo+U*JHNPvH(t@8sEGrS!cG*U8Cio^fU_urUhR? zQrIrsH`F^VtpC#7xr`irO1&~bnqBvnImxN zIU%XZ=@heh;OsfE&dfTFK+Hi`W;UI4z|8{I2xw0P4lZ-)d#DUjGa&YEl zPgWasX<^Ki*G=0uwS50`NoQHhLS6GrEORt$9^R=2{8&K9>#!OE*;x05qyt+;bQ!FW zt>7##3vS!!sYRFyi^8$#bRv?E0PFr;=l;Fs4OINyhb|nSc6Af~HlOuYVf_~(C53zg z{|?(<^MKPC9vB?#-a5A;_-gR_;cwRV^M3h4VKGRDG*_nOLa~sar%$ut0lBBm_EX#B b+{M!Vg!*Pj*Wjg*gI@l9*|!qg$LRbM3K-Aa literal 0 HcmV?d00001 diff --git a/tests/fixtures/files/invalid-tile-type.mbtiles b/tests/fixtures/files/invalid-tile-type.mbtiles new file mode 100644 index 0000000000000000000000000000000000000000..60aa785344012d147ce88626edf00d524ef149b0 GIT binary patch literal 5632 zcmeGfO>bL8@NHfaJ8?@@swj%8mgV(8qq@e9<1bD)j$7K;X^7KS6{@VY-^SkhdDngK zIf;o{sZ>Y^^?=|A5)vHv0bDrrR-yg?gt&9((gUi*g_(VJ<3vfzg$j|Zy>DlBW@l$- z=IzX@d{`1br|ZgXGoL1n5rYupeM$|(n1UXGJ_daR`ULb*=z~TEz)Qv{BPM@B0fhX8 zY!kuok`+g!*Jy$9vE#5?#Q4r2zmRXqm*i8jMQ)O}$w^`wzryr?pe2r(vt`z3RDI!a zui9i@b8)_yh$rKVsZ1u7$j2AvQ~9}UK9iVBXNvRbh3sdM$a^eMaJ9i*STr9pXX-W8m9}>= zF_(-%Pvzp#i^)tZo5;qqsc0fIAIqeZ@wrSim5U`)bD4Pht#NZESLf2_?!`)dCUoH0itAgt#LZcj1sZHG%F!Vm~NGH5^3C7&2mLPfnUMB#7II z6XrQG6ec4`3c_z3!61G^Z<@1Y_nUMK-+w~>G~j>K4vhuE<~RwTFpRUTe`L(%!@2#? zSmKZ#G&~0WCuf;?oIE#73>B1Re2x;>z@2&$ieU){6rSa-OWHm8>Oz?BK@9QuY6IPU3 zLkR?{Bb@5zA3)=X1sMpiG+^t3J2qBS+ZJkyODeuAWW%~g@6p>)D=;qe10G#~6`+vv zfin@cb_%lX07d~pWn)FHdY>_!jam(^KySB)3@dOXALz`7t!iDL4@rD6r^~$U9Ufe$ zRopEXvn`UmKcOkXWE~YKfJzO!t07Tm4P3U!TFk|fODHiA=+>Bk@iLcu16O^ZWCLS>E&-tN zuIDS<=8EStS?f9gUPUoly{oFpq~u<$>mv6O3mBFJ-#}DI58gMFyQG{Jgf~DXPz9AO ziMB?Sw6**a9#b|r6!btXGuwrN904gDWGr(BN)}h5?aF*h)DJpny!X?%Y2hAa;G_fZz&huu)=EF;2n<}|N?WJ`W@$9FX)EAb)(L!G@m0OmR1U^WrDsd6YURoIimtG( zgR<6`=x|hQ4eqHHd@LZyo3I-Ks@2m>%7vpMbOW?d-N2VO1eX#mbrY7trr@?HU5piC zF!kgc_sQe-T@?KDBM;nX-M)vvd;Rx)*KnLXf{*)zVcdtubNIcgErrIXtGR=2nmxCD z)rQZucBh)ye?Ia_WIQyT%aMnP{qjnG_5N!ocU_tX@SNB8{?7GH2bm*C2)JrbwizN| zqJh>=QbPoN&C?*!=nb?Soowh(y6LzO^=^;0cyBoC9gsq1CuhO*{_D<6NA>@!0{;I< b50gRtkKg|UYC!lOwL@fq5PkqmoMnFl)c+~) literal 0 HcmV?d00001