From 62a13fd132d257c0997f4cca27d4bf89f11b118f Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 28 Sep 2023 21:29:43 -0400 Subject: [PATCH 1/2] Fix mbtiles validation, CI, and logging --- Cargo.lock | 1 + justfile | 1 + martin-mbtiles/Cargo.toml | 3 ++- martin-mbtiles/src/bin/main.rs | 33 ++++++++++++++--------- martin-mbtiles/src/mbtiles.rs | 33 +++++++++++------------ tests/expected/auto/catalog_auto.json | 6 +++++ tests/expected/generated_config.yaml | 1 + tests/expected/mbtiles/copy_diff.txt | 1 + tests/expected/mbtiles/validate-bad.txt | 3 +++ tests/expected/mbtiles/validate-fix.txt | 3 +++ tests/expected/mbtiles/validate-fix2.txt | 3 +++ tests/expected/mbtiles/validate-ok.txt | 3 +++ tests/fixtures/files/bad_hash.mbtiles | Bin 0 -> 14848 bytes tests/pg_table_source_test.rs | 2 +- tests/test.sh | 13 +++++++++ 15 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 tests/expected/mbtiles/validate-bad.txt create mode 100644 tests/expected/mbtiles/validate-fix.txt create mode 100644 tests/expected/mbtiles/validate-fix2.txt create mode 100644 tests/expected/mbtiles/validate-ok.txt create mode 100644 tests/fixtures/files/bad_hash.mbtiles diff --git a/Cargo.lock b/Cargo.lock index a09f224f3..0fadd1617 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1799,6 +1799,7 @@ dependencies = [ "actix-rt", "anyhow", "clap", + "env_logger", "futures", "log", "martin-tile-utils", diff --git a/justfile b/justfile index 503a4df79..ba9bfdd6a 100644 --- a/justfile +++ b/justfile @@ -110,6 +110,7 @@ test-int: clean-test install-sqlx # Run integration tests and save its output as the new expected output bless: start clean-test + rm -rf tests/temp cargo test --features bless-tests tests/test.sh rm -rf tests/expected diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index 3392c25a9..fd746f043 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -12,7 +12,7 @@ rust-version.workspace = true [features] # TODO: Disable "cli" feature in default builds default = ["cli", "native-tls"] -cli = ["dep:anyhow", "dep:clap", "dep:tokio"] +cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:tokio"] # One of the following two must be used native-tls = ["sqlx/runtime-tokio-native-tls"] rustls = ["sqlx/runtime-tokio-rustls"] @@ -30,6 +30,7 @@ tilejson.workspace = true # Bin dependencies anyhow = { workspace = true, optional = true } clap = { workspace = true, optional = true } +env_logger = { workspace = true, optional = true } serde_yaml.workspace = true sqlite-hashes.workspace = true tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 5a7b0f542..4df966ce1 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -1,11 +1,10 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; +use log::{error, LevelFilter}; use martin_mbtiles::{ apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions, }; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::{Connection, SqliteConnection}; #[derive(Parser, PartialEq, Eq, Debug)] #[command( @@ -73,9 +72,23 @@ enum Commands { } #[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); +async fn main() { + env_logger::builder() + .filter_level(LevelFilter::Info) + .format_indent(None) + .format_module_path(false) + .format_target(false) + .format_timestamp(None) + .init(); + if let Err(err) = main_int().await { + error!("{err}"); + std::process::exit(1); + } +} + +async fn main_int() -> anyhow::Result<()> { + let args = Args::parse(); match args.command { Commands::MetaAll { file } => { meta_print_all(file.as_path()).await?; @@ -109,8 +122,7 @@ async fn main() -> anyhow::Result<()> { async fn meta_print_all(file: &Path) -> anyhow::Result<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(true).await?; let metadata = mbt.get_metadata(&mut conn).await?; println!("{}", serde_yaml::to_string(&metadata)?); Ok(()) @@ -118,8 +130,7 @@ async fn meta_print_all(file: &Path) -> anyhow::Result<()> { async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(true).await?; if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? { println!("{s}"); } @@ -128,8 +139,7 @@ async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> { async fn meta_set_value(file: &Path, key: &str, value: Option) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(false).await?; mbt.set_metadata_value(&mut conn, key, value).await } @@ -139,8 +149,7 @@ async fn validate_mbtiles( update_agg_tiles_hash: bool, ) -> MbtResult<()> { let mbt = Mbtiles::new(file)?; - let opt = SqliteConnectOptions::new().filename(file).read_only(true); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = mbt.open_with_hashes(!update_agg_tiles_hash).await?; mbt.check_integrity(&mut conn, check_type).await?; mbt.check_each_tile_hash(&mut conn).await?; if update_agg_tiles_hash { diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index 6a1c37242..574cf0035 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -435,7 +435,9 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + let filepath = self.filepath(); if integrity_check == IntegrityCheckType::Off { + info!("Skipping integrity check for {filepath}"); return Ok(()); } @@ -452,13 +454,14 @@ impl Mbtiles { if result.len() > 1 || result.get(0).ok_or(FailedIntegrityCheck( - self.filepath().to_string(), + filepath.to_string(), vec!["SQLite could not perform integrity check".to_string()], ))? != "ok" { return Err(FailedIntegrityCheck(self.filepath().to_string(), result)); } + info!("{integrity_check:?} integrity check passed for {filepath}"); Ok(()) } @@ -466,16 +469,17 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + let filepath = self.filepath(); let Some(stored) = self.get_agg_tiles_hash(&mut *conn).await? else { - return Err(AggHashValueNotFound(self.filepath().to_string())); + return Err(AggHashValueNotFound(filepath.to_string())); }; - let computed = calc_agg_tiles_hash(&mut *conn).await?; if stored != computed { - let file = self.filepath().to_string(); + let file = filepath.to_string(); return Err(AggHashMismatch(computed, stored, file)); } + info!("The agg_tiles_hashes={computed} has been verified for {filepath}"); Ok(()) } @@ -486,23 +490,15 @@ impl Mbtiles { { let old_hash = self.get_agg_tiles_hash(&mut *conn).await?; let hash = calc_agg_tiles_hash(&mut *conn).await?; + let path = self.filepath(); if old_hash.as_ref() == Some(&hash) { - info!( - "agg_tiles_hash is already set to the correct value `{hash}` in {}", - self.filepath() - ); + info!("agg_tiles_hash is already set to the correct value `{hash}` in {path}"); Ok(()) } else { if let Some(old_hash) = old_hash { - info!( - "Updating agg_tiles_hash from {old_hash} to {hash} in {}", - self.filepath() - ); + info!("Updating agg_tiles_hash from {old_hash} to {hash} in {path}"); } else { - info!( - "Initializing agg_tiles_hash to {hash} in {}", - self.filepath() - ); + info!("Creating new metadata value agg_tiles_hash = {hash} in {path}"); } self.set_metadata_value(&mut *conn, "agg_tiles_hash", Some(hash)) .await @@ -550,7 +546,10 @@ impl Mbtiles { v.get(0), v.get(1), )) - }) + })?; + + info!("All tile hashes are valid for {}", self.filepath()); + Ok(()) } } diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index 1de39eb3b..54051c79d 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -8,6 +8,12 @@ "content_type": "application/x-protobuf", "description": "autodetect.auto_table.geom" }, + "bad_hash": { + "content_type": "application/x-protobuf", + "content_encoding": "gzip", + "name": "Major cities from Natural Earth data", + "description": "Major cities from Natural Earth data" + }, "bigint_table": { "content_type": "application/x-protobuf", "description": "autodetect.bigint_table.geom" diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index c51f1f02f..930b6b036 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -141,6 +141,7 @@ pmtiles: mbtiles: paths: tests/fixtures/files sources: + bad_hash: tests/fixtures/files/bad_hash.mbtiles geography-class-jpg: tests/fixtures/files/geography-class-jpg.mbtiles geography-class-jpg-diff: tests/fixtures/files/geography-class-jpg-diff.mbtiles geography-class-jpg-modified: tests/fixtures/files/geography-class-jpg-modified.mbtiles diff --git a/tests/expected/mbtiles/copy_diff.txt b/tests/expected/mbtiles/copy_diff.txt index e69de29bb..4fc7c848d 100644 --- a/tests/expected/mbtiles/copy_diff.txt +++ b/tests/expected/mbtiles/copy_diff.txt @@ -0,0 +1 @@ +[INFO ] Creating new metadata value agg_tiles_hash = C7E2E5A9BA04693994DB1F57D1DF5646 in tests/temp/world_cities_diff.mbtiles diff --git a/tests/expected/mbtiles/validate-bad.txt b/tests/expected/mbtiles/validate-bad.txt new file mode 100644 index 000000000..242a6aaaa --- /dev/null +++ b/tests/expected/mbtiles/validate-bad.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for ./tests/fixtures/files/bad_hash.mbtiles +[INFO ] All tile hashes are valid for ./tests/fixtures/files/bad_hash.mbtiles +[ERROR] Computed aggregate tiles hash D4E1030D57751A0B45A28A71267E46B8 does not match tile data in metadata CAFEC0DEDEADBEEFDEADBEEFDEADBEEF for MBTile file ./tests/fixtures/files/bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix.txt b/tests/expected/mbtiles/validate-fix.txt new file mode 100644 index 000000000..ded275f11 --- /dev/null +++ b/tests/expected/mbtiles/validate-fix.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles +[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles +[INFO ] Updating agg_tiles_hash from CAFEC0DEDEADBEEFDEADBEEFDEADBEEF to D4E1030D57751A0B45A28A71267E46B8 in tests/temp/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-fix2.txt b/tests/expected/mbtiles/validate-fix2.txt new file mode 100644 index 000000000..250ba58e6 --- /dev/null +++ b/tests/expected/mbtiles/validate-fix2.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for tests/temp/fix_bad_hash.mbtiles +[INFO ] All tile hashes are valid for tests/temp/fix_bad_hash.mbtiles +[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for tests/temp/fix_bad_hash.mbtiles diff --git a/tests/expected/mbtiles/validate-ok.txt b/tests/expected/mbtiles/validate-ok.txt new file mode 100644 index 000000000..2220bb36d --- /dev/null +++ b/tests/expected/mbtiles/validate-ok.txt @@ -0,0 +1,3 @@ +[INFO ] Quick integrity check passed for ./tests/fixtures/files/zoomed_world_cities.mbtiles +[INFO ] All tile hashes are valid for ./tests/fixtures/files/zoomed_world_cities.mbtiles +[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/files/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/bad_hash.mbtiles b/tests/fixtures/files/bad_hash.mbtiles new file mode 100644 index 0000000000000000000000000000000000000000..33943e69916a1187237dfd4b0dd39d6309b054ab GIT binary patch literal 14848 zcmeHO33wdUk$$a_N3ssfmJj*%XslZ!sqX`9O!w<<>zI*^Fp-jWwE)XGS)T zF^NMCl8`Jo$!^#vArOcOM=p}U2D9MUfFTRm_9q#{2y|gypU2#Ptg(9(@@U_X^QIOe=;7}@V z9q&j*qT%F@y^&Pkj=o^BZwGV1iCH_Sf{GO@sDtxT!B8|j5KaYqf~lbWI}dL%)Xj!x zv#R+GkA2ot7lS)IsqnQaPv?fsp3W_Pzh~p7wt%|nN>7JzrO&e`7#&LAP~Vd0uD~mD zjn97N4#uWjX+gVPr9c#;I}r6WbsKev`Z#qhwT-Hz=20&66FB{6-{b&O>Z%U}dwVld zCIZH)X<%A6bVJoO!!R%WePeM^{=%E*?@GpF`@MU@-KltDM>M!EoJe{v@m%HE@9{=@ zfbw=n;QIdjA#22`hbcf`2?kz6Ih8BSGiqFlb(R z@hsP}vVq{WABx8Z#NvE%5Q&iky4VE=z&ja<$5=McwR%fWIN67Ao%C&F{v7L*#xrllc6;m+^{=Y%JLj8i8q<%=< zLw%L{d+M{)!9TnD|9S=ha$ekB?y9G>}Ty z(*`q(+(@ju2;@_ib~a>Ql#~>?mQZO^o?YfzO*t$lk6cPyq??dwlP(~>c^-8yk@E`h z&2Lh#QfH`_s441~)X%{uOi&}#aq3a(N7N6f@1$>oI|uU%D2h6`IN&a>S@oGh@-WJC zeV)DhnyFC7gm1*(^wYkLH-|<##sXWWem(W(NXyp2v5pPnZB2`&ZnR6g~^48LCwKlZ1Ia`fz}FK=gnAuIEeAQ`Hpu(YYk8aYs4Zu2@;6`D4a zc^1Q!z_8ZAkp0$73}2cm?HubGY-|UvzRrmX2h$%v+PVq_`fHvPfT_S~q9B{HqMAIb z>5Q&mmcb0KnKUgLG{2j&m@WmTS#rJ4VhS%tI)21-p43*v(xIyRyZwo3j9RrE_6dLLz%HQCm@d9_%UVHQ1v?)N9l;F#jK?9;Uua z9j3lQeV+Oh^ZgJb6#^8d6sm=)rj|i`C_w*-UPmvZe}y=45*WxJtI_OPU?!Bix}m|Q>gsK(y3VGS)n=%w+8Uc$R-K_%H&xlxlGPc?+vv5a zhDw{N_t;e3Dw|rhGDB6?ugFlIhUGR@zs#m8mu9F{4NGjQc5#MUSyPdrmU|avsKv_` zW~ho~3o_KgCG#`Xg39s?HGj#x3{_q`*QPvWHnn0-hMHGbYEv~OHdR%eq2|^V*;LK! z3{_TLn4#uW71-2jcZMphot2@AYw|NxQG?5(K-2c%y$zM60LNUY22neyCs7SmOZ}dh z!iC$x>sA@uB(ao-dvYR^z($G@lfVP#3xz+%2(vMU~>Q|$B429kU0Q;_4Oc@b$@O6&+^*@ z|K(8uL>-0Q-?sNBfyAtWg%uTr2iNb3g!fv8Z|ySkVQZVQ%`)($NA2=-8GZwA_MD#$ zIczU;+A?i((}qC$4d;`M_nDc%2hS#8((Chh9cY1|V{$43KH!ihM``#0M1&jHWXK5tqA!BqMl3xHz)(kZx;@p;b!^8IO=ydaHiy*_Vu z=0;KWd3(cg$XgQo$PF7Ii1%Fn|B~d&q;<5VPM)pcc85Opn^< zwGt}pVXxZL69GMif}tRAQwNfOexgiiH^V<0aeqQO|ox;)g~4|ETl)PlWzJ;5{; z>kUTZiLiAx*4rQNCsa7Hi-?F&iD)E7hUm#Bti7y#lc7-Zv7q%dp5`S zl6!G55$q18uO=av492?205A-xw`a(rBHh7W@|Yg%4<^X(a18JzA%;T12>f0jj)nJ- zr&{B&USjZ`4j`=Kp=2sf*t8{6!B}W03S>KZjWxXXcwaCU3nxQE337KwFnvQuB)o?l zMq)|ux8%JY@n}DIVxasm6f#(UWWYL%$9k;y`^jgVZ z7I0TMZb_&M#M0YmiM9(=8yrmQyek#&?(d67iOqEF>xo%I?K<}LxMyQN^`{(aU>@dSJ-p|-*(;zMa&ZjFRvF(RpL@F*|^wfQ`HGj4`c z$A%NfjtuN3FMj-{B&g4O^#Stlz)a4K`2Tz)AjtL>qr*U4%=S0@pPl0T|KR_3&a0;L zobBC@w$DO?yM8&&fiAJ68ydqvnqx4mj76OnM8>2IQPm8d=jpWm*CD9=tU}*H>n=w9 zj(Y|Qt&F8S;4Z6nq)DglpD9iHRp_k~EltfMlk*)3(y=ug+-Su^=l;|JuT^DEpam0h zGf~kMLl#8LFtnmFGKO7yjDq#uJSmjwkVO36~X~V?3-F8McXSIVNo6gAor$z$zpYTs?8=UZc z)4}r0p`TY3pxgf8k-LdTcu`{%S!Y#)=Qvi@O$oF^%Q9=w0(?9lon^6{8*rC8GtX1s zoJpk5hmvjMjahl_>DnJ{C`7-1dw5?nkti!Lf@Vq*4<$2IHBI~6=wtYRqAp6BJTQBjU6v$s%&2KePS5`%!4J2V zB05POBkW`yv@I}L76nD;MMGj}gXT0E${i9fib7id^+f;Gki*yKRQ|!@xz@ls1MYIC zT^7wW70Rc|3pe;CPK)A2g*IdjfvI6v{y ztpnX|_AVHM$%ux?ftNBxo`HEO$~tG73WsS~*Lc~CmRW;=NmA}{Dkaw>8Qn11 z0_HX?q%-u|Cri;&@kRB--moEvn$B~qq^g=^Y64a?6HAIFVa4D@ylaldtpvEOaB{n6 zhAA0(G4$%lSir$+d*$iJXCeQ>XI7B<3|1H&8=THcg1|GHEa4 ze<`AFrYfO=b16J>vD@GRYot&GtXSrd%FI>37p96XJ0kC>^n91E-Erf#k6v-(rw*#0 z`eO37-&O?CWyO12NEHxE8f{{kl~_?yC6?H&$!R>T2`p_$vY41}aahzz5&}EpL|v6d zaiHAR$AU@+&z#f0)7S3r^f}o*`Q}Ggpf`KI`y9~+sb{J*Z!)Y7pCoaT$uppKo~E&4 zDzX`yXR%ulaL;#s;;DOQFg2J_g`H!KBmOLRbFOmj`h0ZX;B^N{O^_E=9h(aHPDO@l zqd>C^^i04AMGV%!m}&j5MbxLM73f~H79P3yExOnmEX=}HPM0vV*MC0rdca>l>YsOB z6KCc=`~-zw=quVzW+AJ?ESCguR3;;cG$X*QRIv)^g%@=OD}xmlH?X&rPS2Rj-V&X@ zPXB1zSVyOGF5bQ`Fdy9(W}hJT2JiwfkOVG9;CWf*Ii5BJh{*yrHBQ1p|00Vc#F^z@ zr|xsanW46^PM@>lymOyXf{t$O4v;v*3IY~c6a0yyDW+-Ax`ky)T_pZLNG-Rm2^ou^@+!n(Q8ck3835FDeb*{m`QEG`IivDl49y;$3@kkFcB0S!JDrEt-1Sl; zQ9dI9^n>gI+@Xne8ge558XA}YW6&bUYO$3TFYw6TRZc_8;gQcyj5SWQ`kYQ@`0EcI zpNl?u>P^cdgC~<=Izj}HY1ZJOxdo6D+Sw+<7_7oD(G|8ND=$Wp$qnNjO-@N(_v9Zh zuRurDHw%d*c})={ofa8hGC0Oi1%rX<#PG7FvpT1%Y5ijay`F0HeT3hG^5a$3fFUVc z-Q*Nd4rfr(JhEY8;>6^LLnG%Ndp4Mdj(%BqizHKwyEOAdN+I)(vbhBY91 zNlJY6G{dYInS)`f)7QGCaHPeVb{<)MZCepyp1$o~!q5;@=*&qH1G7@*6%9jF<{%4D zL{^t+L5zBBm3SMSUzo$VlkH8TBV&#v_jEP$+F$MXJ!90jTC(FQ0G`yq*`;wn)T)A-17hW z1sSO+ikhS`yds$(8bPKN5RPUj5FZVd=e0zw#SFk-byj?wY0d-qdv$cOb?jtEQ)fd{ zyCXWD2_FgNp<|n_YbD^1V+~z2cvZ%#PGe32ST3s&oi#|6dC3%_HMZibE(YKveIu>o zjSkNreg4aLxX|fWZ~O{TJcKY+&=o-9;Dvc?D3H_w7}G&FA_N>MQEh)=)%*Cub3J<> z&qsG&Ua^^cp#&8aLsFr;uJDYkK%bpfB$b8i4~9Y8`JYDce--5a8a(nYx7|K#)X=o9 zaR$d+C6YbeFQ?QOR$zFu4I5F$n4x7)In2QXS(p zK?S&MXre6dYOvS=)TqvykTY2xN&hDNO^(SwT=_`bLNu~&$8SiqF(p=G6%AGn8G!s? zfrDvc8bXZNJoJdxXc%Xexy+!>8_2U0>{~z9c>TmsjYYoI-PXC{R6)ky% zWM#%|SY}d09YVEeJECr+uBTGeF6tU;E9Iv&N~W5rDrzY;mvWRSr zlz1Pk4Ez%M9Qqjg5E?`~(RNrjbOzQ8{T$W{9f!3--*NL*>t@?a5ct~5GfNPxvz9ox zm6v7CvDIsB%Cp9%R$iK+81E$+s-;e`sVX@`HC0PCa~Z0>hPA0G zCPUR#(HW|?rX@qw_?m61vB{V|%+*Vl ztj|!&VlzX@HAaRKYxN8zRAZZ}(rn6`!GD1N3I2OO_@B1?7Gy1TStZ#V5UF_dKEHot z?s-f;T$S+8LAN~e#5@uyMZttRyn?aBL775_$gD{avH%OvScYQuK#Ls!YMrHkoJE;- z|72@hR^{P)!-tE}x{ZH$lmq~%GwD3kn@A}DdJUS%0bVvScxF*yFe^u!?T@SRT?GDn zeY~Y}qQwC^&fWFMNeX@P!Pv`b{AWNBfa4?;0-C5nNX8&XGB^c_fsi3_ zJV!$^NCQCRWGTwo8u2;%GC7mvrtv`gm;(Tu+nH=#g62Q>z*9sckOyI1F(q1}d4|?Z z5mYa6kOvAP2#%qntj(=H>(g=>ciVVfyA%JPc|JNd3%$MP6L%AC3VaH*H&`gmicsFv zY0{!2h!?P-EMmQ%u?1P5HI;G*a;W}-PYbffa#TUReAp5s3;hz4!wl4ZSe1qZ09IP+ zunG~1&+w6%`XBWFZhmk5KiL{9AhDLLvF1Wz6O$vIK4&5R+#TGpa&$%6-LI302pB~b zuq<#iuL`gvQ&DA;ST3YRSdk1SE?Mj#)ojX2IC5luzk`0mVB7e}35VvlAC>|IC_b(n zAekRXKvH->c>`PjyCNIDea7|$pl~bV>FB`l8#clXlNOwg6#{`QZ;6LBR;J?B)=(2Q8jUG z7NN7lr@lz=pD9aNXH1PXAl-m`T7{YqizTQ(h|p_gGWCBoUH|_cg1*?h+~R9&39(t7 zm>d!+nE(m-PGm{wi09$Q=AxIcdf*l!Ax-8XTP0O-NLwVB`vCn-Mx{kq8w`b6X5dnb zA($My(&>zInOr**WgDC(cRKv&=StDjKY2wUCdWw%2NMxsJ_Cz@VQntdgc(hRxPvu; z)|r7zY<5hR1>|727c5|MV$zWV9=YbZujM0r_Tisdp+=XW(I`Ps;TatYjG6%@23e({ zy1{dp(_)IP|CWosOZ$HvLH}xZD;IpOq!&6w&zD=vap%J`UfN z{Ol^ky?E<>%L#zXf)pHlBMcIH7qkWqQbQzFIY|Dn9$RONh|lslIV|pXCnh_#opqOM zN6K%~{3Rvr_YHskV`W#m(4m57CyBwSD*W*V6k=hO1h4Zd$I$|(=$Zz#KVBC&J$jkN z519{Y0^;WVoU*gPOt_+!5k;51`}1Mxn((=9NbSum<0S1v3->8AN@ADnqv;G9TV~i?!OqU$@xlesc(a z&=;7PfD(l>K>avz(-O4cp|9RYgfByBho*Ij(?AHE%);Mt8Bm!Z%~4T@K6P}xEsjgF zioH3+QT#3cNQZO1&+s+7q0)QpKR$Gnh(i_?nA5rle_aB77YSbSUT;_o`G8w?Ld!&s&%5*&UBh4c%H78&9$$LswySS#Io^1*?)dS?zIJ?6`OF)i wt@^^r!e9OHOOH%ZDDmQnXH4J^!Hu0jkz>Z~y=R literal 0 HcmV?d00001 diff --git a/tests/pg_table_source_test.rs b/tests/pg_table_source_test.rs index 4207138d4..5bba0b85b 100644 --- a/tests/pg_table_source_test.rs +++ b/tests/pg_table_source_test.rs @@ -113,5 +113,5 @@ async fn table_source_schemas() { functions: false "}); let sources = mock_sources(cfg).await.0; - assert_eq!(sources.keys().collect::>(), vec!["MixPoints"],); + assert_eq!(sources.keys().collect::>(), vec!["MixPoints"]); } diff --git a/tests/test.sh b/tests/test.sh index 846e1a3de..6d9fdde17 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -293,6 +293,19 @@ if [[ "$MBTILES_BIN" != "-" ]]; then $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt" $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt" + $MBTILES_BIN validate ./tests/fixtures/files/zoomed_world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-ok.txt" + + set +e + $MBTILES_BIN validate ./tests/fixtures/files/bad_hash.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-bad.txt" + if [[ $? -eq 0 ]]; then + echo "ERROR: validate with bad_hash should have failed" + exit 1 + fi + set -e + + cp ./tests/fixtures/files/bad_hash.mbtiles "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" + $MBTILES_BIN validate --update-agg-tiles-hash "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix.txt" + $MBTILES_BIN validate "$TEST_TEMP_DIR/fix_bad_hash.mbtiles" 2>&1 | tee "$TEST_OUT_DIR/validate-fix2.txt" # Create diff file $MBTILES_BIN copy \ From c00f12f8f686d7a239a6049675ce28fa053a7a28 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 28 Sep 2023 21:53:45 -0400 Subject: [PATCH 2/2] moved test files --- justfile | 2 +- martin-mbtiles/src/mbtiles.rs | 18 +++--- martin-mbtiles/src/tile_copier.rs | 54 +++++++++--------- tests/config.yaml | 2 +- tests/expected/auto/catalog_auto.json | 6 -- tests/expected/generated_config.yaml | 39 +++++++------ tests/expected/given_config.yaml | 2 +- tests/expected/mbtiles/copy_diff2.txt | 1 + tests/expected/mbtiles/validate-ok.txt | 6 +- .../{invalid => }/invalid-tile-format.mbtiles | Bin .../files/{invalid => }/invalid.mbtiles | Bin .../invalid_zoomed_world_cities.mbtiles | Bin .../geography-class-jpg-diff.mbtiles | Bin .../geography-class-jpg-modified.mbtiles | Bin .../geography-class-jpg.mbtiles | Bin .../geography-class-png-no-bounds.mbtiles | Bin .../geography-class-png.mbtiles | Bin .../fixtures/{files => mbtiles}/json.mbtiles | Bin .../uncompressed_mvt.mbtiles | Bin .../fixtures/{files => mbtiles}/webp.mbtiles | Bin .../{files => mbtiles}/world_cities.mbtiles | Bin .../world_cities_diff.mbtiles | Bin .../world_cities_modified.mbtiles | Bin .../zoomed_world_cities.mbtiles | Bin tests/fixtures/{files => pmtiles}/png.pmtiles | Bin ...stamen_toner__raster_CC-BY+ODbL_z3.pmtiles | Bin .../fixtures/{files => pmtiles}/webp2.pmtiles | Bin tests/mb_server_test.rs | 8 +-- tests/pmt_server_test.rs | 4 +- tests/test.sh | 18 +++--- 30 files changed, 79 insertions(+), 81 deletions(-) rename tests/fixtures/files/{invalid => }/invalid-tile-format.mbtiles (100%) rename tests/fixtures/files/{invalid => }/invalid.mbtiles (100%) rename tests/fixtures/files/{invalid => }/invalid_zoomed_world_cities.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/geography-class-jpg-diff.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/geography-class-jpg-modified.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/geography-class-jpg.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/geography-class-png-no-bounds.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/geography-class-png.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/json.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/uncompressed_mvt.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/webp.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/world_cities.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/world_cities_diff.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/world_cities_modified.mbtiles (100%) rename tests/fixtures/{files => mbtiles}/zoomed_world_cities.mbtiles (100%) rename tests/fixtures/{files => pmtiles}/png.pmtiles (100%) rename tests/fixtures/{files => pmtiles}/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles (100%) rename tests/fixtures/{files => pmtiles}/webp2.pmtiles (100%) diff --git a/justfile b/justfile index ba9bfdd6a..a8aacc642 100644 --- a/justfile +++ b/justfile @@ -217,7 +217,7 @@ git-pre-push: stop start # Update sqlite database schema. prepare-sqlite: install-sqlx mkdir -p martin-mbtiles/.sqlx - cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests + cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/mbtiles/world_cities.mbtiles -- --lib --tests # Install SQLX cli if not already installed. [private] diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index 574cf0035..a3d350dc9 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -609,7 +609,7 @@ mod tests { #[actix_rt::test] async fn mbtiles_meta() { - let filepath = "../tests/fixtures/files/geography-class-jpg.mbtiles"; + let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"; let mbt = Mbtiles::new(filepath).unwrap(); assert_eq!(mbt.filepath(), filepath); assert_eq!(mbt.filename(), "geography-class-jpg"); @@ -617,7 +617,7 @@ mod tests { #[actix_rt::test] async fn metadata_jpeg() { - let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; let metadata = mbt.get_metadata(&mut conn).await.unwrap(); let tj = metadata.tilejson; @@ -634,7 +634,7 @@ mod tests { #[actix_rt::test] async fn metadata_mvt() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let metadata = mbt.get_metadata(&mut conn).await.unwrap(); let tj = metadata.tilejson; @@ -665,7 +665,7 @@ mod tests { #[actix_rt::test] async fn metadata_get_key() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap(); assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706"); @@ -725,15 +725,15 @@ mod tests { #[actix_rt::test] async fn detect_type() { - let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::Flat); - let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::FlatWithHash); - let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); assert_eq!(res, MbtType::Normalized); @@ -744,7 +744,7 @@ mod tests { #[actix_rt::test] async fn validate_valid_file() { - let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await; + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; mbt.check_integrity(&mut conn, IntegrityCheckType::Quick) .await @@ -754,7 +754,7 @@ mod tests { #[actix_rt::test] async fn validate_invalid_file() { let (mut conn, mbt) = - open("../tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles").await; + open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await; let result = mbt.check_agg_tiles_hashes(&mut conn).await; assert!(matches!(result, Err(MbtError::AggHashMismatch(..)))); } diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index 6827f410f..8e8ced1b5 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -603,14 +603,14 @@ mod tests { #[actix_rt::test] async fn copy_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_flat_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, Flat).await } #[actix_rt::test] async fn copy_flat_from_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_flat_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); @@ -619,7 +619,7 @@ mod tests { #[actix_rt::test] async fn copy_flat_from_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_flat_from_normalized_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, Some(Flat), Flat).await @@ -627,14 +627,14 @@ mod tests { #[actix_rt::test] async fn copy_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from("file:copy_flat_with_hash_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, FlatWithHash).await } #[actix_rt::test] async fn copy_flat_with_hash_from_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_flat_with_hash_from_flat_tables_mem_db?mode=memory&cache=shared", ); @@ -643,7 +643,7 @@ mod tests { #[actix_rt::test] async fn copy_flat_with_hash_from_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from( "file:copy_flat_with_hash_from_normalized_tables_mem_db?mode=memory&cache=shared", ); @@ -652,14 +652,14 @@ mod tests { #[actix_rt::test] async fn copy_normalized_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_normalized_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, None, Normalized).await } #[actix_rt::test] async fn copy_normalized_from_flat_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_normalized_from_flat_tables_mem_db?mode=memory&cache=shared"); verify_copy_all(src, dst, Some(Normalized), Normalized).await @@ -667,7 +667,7 @@ mod tests { #[actix_rt::test] async fn copy_normalized_from_flat_with_hash_tables() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles"); let dst = PathBuf::from( "file:copy_normalized_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); @@ -676,7 +676,7 @@ mod tests { #[actix_rt::test] async fn copy_with_min_max_zoom() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) @@ -686,7 +686,7 @@ mod tests { #[actix_rt::test] async fn copy_with_zoom_levels() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) @@ -697,11 +697,11 @@ mod tests { #[actix_rt::test] async fn copy_with_diff_with_file() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let dst = PathBuf::from("file:copy_with_diff_with_file_mem_db?mode=memory&cache=shared"); let diff_file = - PathBuf::from("../tests/fixtures/files/geography-class-jpg-modified.mbtiles"); + PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); @@ -744,10 +744,10 @@ mod tests { #[actix_rt::test] async fn ignore_dst_type_when_copy_to_existing() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from( "file:ignore_dst_type_when_copy_to_existing_mem_db?mode=memory&cache=shared", ); @@ -761,8 +761,8 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_abort_mode() { - let src = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); - let dst = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); + let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort); @@ -775,10 +775,10 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_override_mode() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared"); @@ -804,10 +804,10 @@ mod tests { #[actix_rt::test] async fn copy_to_existing_ignore_mode() -> MbtResult<()> { - let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); // Copy the dst file to an in-memory DB - let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared"); @@ -858,7 +858,7 @@ mod tests { #[actix_rt::test] async fn apply_flat_diff_file() -> MbtResult<()> { // Copy the src file to an in-memory DB - let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) @@ -866,11 +866,11 @@ mod tests { .await?; // Apply diff to the src data in in-memory DB - let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles"); + let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/files/world_cities_modified.mbtiles"; + let path = "../tests/fixtures/mbtiles/world_cities_modified.mbtiles"; attach_other_db(&mut src_conn, path).await?; assert!( @@ -886,7 +886,7 @@ mod tests { #[actix_rt::test] async fn apply_normalized_diff_file() -> MbtResult<()> { // Copy the src file to an in-memory DB - let src_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) @@ -894,11 +894,11 @@ mod tests { .await?; // Apply diff to the src data in in-memory DB - let diff_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg-diff.mbtiles"); + let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/files/geography-class-jpg-modified.mbtiles"; + let path = "../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"; attach_other_db(&mut src_conn, path).await?; assert!( diff --git a/tests/config.yaml b/tests/config.yaml index 1f9dc1c66..471f31428 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -162,7 +162,7 @@ postgres: pmtiles: sources: - pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles sprites: paths: tests/fixtures/sprites/src1 diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index 54051c79d..1de39eb3b 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -8,12 +8,6 @@ "content_type": "application/x-protobuf", "description": "autodetect.auto_table.geom" }, - "bad_hash": { - "content_type": "application/x-protobuf", - "content_encoding": "gzip", - "name": "Major cities from Natural Earth data", - "description": "Major cities from Natural Earth data" - }, "bigint_table": { "content_type": "application/x-protobuf", "description": "autodetect.bigint_table.geom" diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index 930b6b036..e8229fd9a 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -133,24 +133,27 @@ postgres: schema: public function: function_zxy_row_key pmtiles: - paths: tests/fixtures/files + paths: + - tests/fixtures/mbtiles + - tests/fixtures/pmtiles sources: - png: tests/fixtures/files/png.pmtiles - stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - webp2: tests/fixtures/files/webp2.pmtiles + png: tests/fixtures/pmtiles/png.pmtiles + stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + webp2: tests/fixtures/pmtiles/webp2.pmtiles mbtiles: - paths: tests/fixtures/files + paths: + - tests/fixtures/mbtiles + - tests/fixtures/pmtiles sources: - bad_hash: tests/fixtures/files/bad_hash.mbtiles - geography-class-jpg: tests/fixtures/files/geography-class-jpg.mbtiles - geography-class-jpg-diff: tests/fixtures/files/geography-class-jpg-diff.mbtiles - geography-class-jpg-modified: tests/fixtures/files/geography-class-jpg-modified.mbtiles - geography-class-png: tests/fixtures/files/geography-class-png.mbtiles - geography-class-png-no-bounds: tests/fixtures/files/geography-class-png-no-bounds.mbtiles - json: tests/fixtures/files/json.mbtiles - uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles - webp: tests/fixtures/files/webp.mbtiles - world_cities: tests/fixtures/files/world_cities.mbtiles - world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles - world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles - zoomed_world_cities: tests/fixtures/files/zoomed_world_cities.mbtiles + geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles + geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles + geography-class-jpg-modified: tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles + geography-class-png: tests/fixtures/mbtiles/geography-class-png.mbtiles + geography-class-png-no-bounds: tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles + json: tests/fixtures/mbtiles/json.mbtiles + uncompressed_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles + webp: tests/fixtures/mbtiles/webp.mbtiles + world_cities: tests/fixtures/mbtiles/world_cities.mbtiles + world_cities_diff: tests/fixtures/mbtiles/world_cities_diff.mbtiles + world_cities_modified: tests/fixtures/mbtiles/world_cities_modified.mbtiles + zoomed_world_cities: tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/expected/given_config.yaml b/tests/expected/given_config.yaml index 033c66283..6ba51282e 100644 --- a/tests/expected/given_config.yaml +++ b/tests/expected/given_config.yaml @@ -159,7 +159,7 @@ postgres: - 90.0 pmtiles: sources: - pmt: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles sprites: paths: tests/fixtures/sprites/src1 sources: diff --git a/tests/expected/mbtiles/copy_diff2.txt b/tests/expected/mbtiles/copy_diff2.txt index e69de29bb..cddb2637d 100644 --- a/tests/expected/mbtiles/copy_diff2.txt +++ b/tests/expected/mbtiles/copy_diff2.txt @@ -0,0 +1 @@ +[INFO ] Creating new metadata value agg_tiles_hash = D41D8CD98F00B204E9800998ECF8427E in tests/temp/world_cities_diff_modified.mbtiles diff --git a/tests/expected/mbtiles/validate-ok.txt b/tests/expected/mbtiles/validate-ok.txt index 2220bb36d..87a15d314 100644 --- a/tests/expected/mbtiles/validate-ok.txt +++ b/tests/expected/mbtiles/validate-ok.txt @@ -1,3 +1,3 @@ -[INFO ] Quick integrity check passed for ./tests/fixtures/files/zoomed_world_cities.mbtiles -[INFO ] All tile hashes are valid for ./tests/fixtures/files/zoomed_world_cities.mbtiles -[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/files/zoomed_world_cities.mbtiles +[INFO ] Quick integrity check passed for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles +[INFO ] All tile hashes are valid for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles +[INFO ] The agg_tiles_hashes=D4E1030D57751A0B45A28A71267E46B8 has been verified for ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/invalid/invalid-tile-format.mbtiles b/tests/fixtures/files/invalid-tile-format.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid-tile-format.mbtiles rename to tests/fixtures/files/invalid-tile-format.mbtiles diff --git a/tests/fixtures/files/invalid/invalid.mbtiles b/tests/fixtures/files/invalid.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid.mbtiles rename to tests/fixtures/files/invalid.mbtiles diff --git a/tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles b/tests/fixtures/files/invalid_zoomed_world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles rename to tests/fixtures/files/invalid_zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg-diff.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg-diff.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg-modified.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg-modified.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles diff --git a/tests/fixtures/files/geography-class-jpg.mbtiles b/tests/fixtures/mbtiles/geography-class-jpg.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-jpg.mbtiles rename to tests/fixtures/mbtiles/geography-class-jpg.mbtiles diff --git a/tests/fixtures/files/geography-class-png-no-bounds.mbtiles b/tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-png-no-bounds.mbtiles rename to tests/fixtures/mbtiles/geography-class-png-no-bounds.mbtiles diff --git a/tests/fixtures/files/geography-class-png.mbtiles b/tests/fixtures/mbtiles/geography-class-png.mbtiles similarity index 100% rename from tests/fixtures/files/geography-class-png.mbtiles rename to tests/fixtures/mbtiles/geography-class-png.mbtiles diff --git a/tests/fixtures/files/json.mbtiles b/tests/fixtures/mbtiles/json.mbtiles similarity index 100% rename from tests/fixtures/files/json.mbtiles rename to tests/fixtures/mbtiles/json.mbtiles diff --git a/tests/fixtures/files/uncompressed_mvt.mbtiles b/tests/fixtures/mbtiles/uncompressed_mvt.mbtiles similarity index 100% rename from tests/fixtures/files/uncompressed_mvt.mbtiles rename to tests/fixtures/mbtiles/uncompressed_mvt.mbtiles diff --git a/tests/fixtures/files/webp.mbtiles b/tests/fixtures/mbtiles/webp.mbtiles similarity index 100% rename from tests/fixtures/files/webp.mbtiles rename to tests/fixtures/mbtiles/webp.mbtiles diff --git a/tests/fixtures/files/world_cities.mbtiles b/tests/fixtures/mbtiles/world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities.mbtiles rename to tests/fixtures/mbtiles/world_cities.mbtiles diff --git a/tests/fixtures/files/world_cities_diff.mbtiles b/tests/fixtures/mbtiles/world_cities_diff.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities_diff.mbtiles rename to tests/fixtures/mbtiles/world_cities_diff.mbtiles diff --git a/tests/fixtures/files/world_cities_modified.mbtiles b/tests/fixtures/mbtiles/world_cities_modified.mbtiles similarity index 100% rename from tests/fixtures/files/world_cities_modified.mbtiles rename to tests/fixtures/mbtiles/world_cities_modified.mbtiles diff --git a/tests/fixtures/files/zoomed_world_cities.mbtiles b/tests/fixtures/mbtiles/zoomed_world_cities.mbtiles similarity index 100% rename from tests/fixtures/files/zoomed_world_cities.mbtiles rename to tests/fixtures/mbtiles/zoomed_world_cities.mbtiles diff --git a/tests/fixtures/files/png.pmtiles b/tests/fixtures/pmtiles/png.pmtiles similarity index 100% rename from tests/fixtures/files/png.pmtiles rename to tests/fixtures/pmtiles/png.pmtiles diff --git a/tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles b/tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles similarity index 100% rename from tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles rename to tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles diff --git a/tests/fixtures/files/webp2.pmtiles b/tests/fixtures/pmtiles/webp2.pmtiles similarity index 100% rename from tests/fixtures/files/webp2.pmtiles rename to tests/fixtures/pmtiles/webp2.pmtiles diff --git a/tests/mb_server_test.rs b/tests/mb_server_test.rs index 912c92a0f..94e1d1dff 100644 --- a/tests/mb_server_test.rs +++ b/tests/mb_server_test.rs @@ -34,10 +34,10 @@ fn test_get(path: &str) -> TestRequest { const CONFIG: &str = indoc! {" mbtiles: sources: - m_json: tests/fixtures/files/json.mbtiles - m_mvt: tests/fixtures/files/world_cities.mbtiles - m_raw_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles - m_webp: tests/fixtures/files/webp.mbtiles + m_json: tests/fixtures/mbtiles/json.mbtiles + m_mvt: tests/fixtures/mbtiles/world_cities.mbtiles + m_raw_mvt: tests/fixtures/mbtiles/uncompressed_mvt.mbtiles + m_webp: tests/fixtures/mbtiles/webp.mbtiles "}; #[actix_rt::test] diff --git a/tests/pmt_server_test.rs b/tests/pmt_server_test.rs index bcb81da41..916601a87 100644 --- a/tests/pmt_server_test.rs +++ b/tests/pmt_server_test.rs @@ -34,12 +34,12 @@ fn test_get(path: &str) -> TestRequest { const CONFIG: &str = indoc! {" pmtiles: sources: - p_png: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + p_png: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles "}; #[actix_rt::test] async fn pmt_get_catalog() { - let path = "pmtiles: tests/fixtures/files/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles"; + let path = "pmtiles: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles"; let app = create_app! { path }; let req = test_get("/catalog").to_request(); diff --git a/tests/test.sh b/tests/test.sh index 6d9fdde17..ab47e712a 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -150,7 +150,7 @@ echo "Test auto configured Martin" TEST_OUT_DIR="$(dirname "$0")/output/auto" mkdir -p "$TEST_OUT_DIR" -ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/files) +ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles) set -x $MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_1.txt & PROCESS_ID=`jobs -p` @@ -289,11 +289,11 @@ if [[ "$MBTILES_BIN" != "-" ]]; then $MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt" $MBTILES_BIN meta-all --help 2>&1 | tee "$TEST_OUT_DIR/meta-all_help.txt" - $MBTILES_BIN meta-all ./tests/fixtures/files/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt" + $MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt" $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt" - $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" - $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt" - $MBTILES_BIN validate ./tests/fixtures/files/zoomed_world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-ok.txt" + $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" + $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt" + $MBTILES_BIN validate ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-ok.txt" set +e $MBTILES_BIN validate ./tests/fixtures/files/bad_hash.mbtiles 2>&1 | tee "$TEST_OUT_DIR/validate-bad.txt" @@ -309,14 +309,14 @@ if [[ "$MBTILES_BIN" != "-" ]]; then # Create diff file $MBTILES_BIN copy \ - ./tests/fixtures/files/world_cities.mbtiles \ + ./tests/fixtures/mbtiles/world_cities.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff.mbtiles" \ - --diff-with-file ./tests/fixtures/files/world_cities_modified.mbtiles \ + --diff-with-file ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff.txt" if command -v sqlite3 > /dev/null; then # Apply this diff to the original version of the file - cp ./tests/fixtures/files/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles" + cp ./tests/fixtures/mbtiles/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles" sqlite3 "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \ -bail \ @@ -328,7 +328,7 @@ if [[ "$MBTILES_BIN" != "-" ]]; then # Ensure that applying the diff resulted in the modified version of the file $MBTILES_BIN copy \ --diff-with-file "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \ - ./tests/fixtures/files/world_cities_modified.mbtiles \ + ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff2.txt"