diff --git a/Cargo.lock b/Cargo.lock index 46fd8cd27..207615c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ [[package]] name = "martin-mbtiles" -version = "0.5.0" +version = "0.6.0" dependencies = [ "actix-rt", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 9893e655d..173ab1b41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ indoc = "2" itertools = "0.11" json-patch = "1.1" log = "0.4" -martin-mbtiles = { path = "./martin-mbtiles", version = "0.5.0", default-features = false } +martin-mbtiles = { path = "./martin-mbtiles", version = "0.6.0", default-features = false } martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" } num_cpus = "1" pmtiles = { version = "0.3", features = ["mmap-async-tokio", "tilejson"] } diff --git a/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json b/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json deleted file mode 100644 index 0848bf78d..000000000 --- a/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT (\n -- Has a \"map\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- \"map\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_id\" AND type = \"TEXT\"))\n --\n ) AND (\n -- Has a \"images\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- \"images\" table's columns and their types are as expected:\n -- 2 columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE ((name = \"tile_id\" AND type = \"TEXT\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) AS is_valid;\n", - "describe": { - "columns": [ - { - "name": "is_valid", - "ordinal": 0, - "type_info": "Int" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - null - ] - }, - "hash": "14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628" -} diff --git a/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json b/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json deleted file mode 100644 index 6e141d9d8..000000000 --- a/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT (\n -- Has a \"tiles\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) as is_valid;\n", - "describe": { - "columns": [ - { - "name": "is_valid", - "ordinal": 0, - "type_info": "Int" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - null - ] - }, - "hash": "177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd" -} diff --git a/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json b/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json deleted file mode 100644 index 6230f16d4..000000000 --- a/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT (\n -- Has a \"tiles_with_hash\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles_with_hash'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles_with_hash\" table's columns and their types are as expected:\n -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).\n -- The order is not important\n SELECT COUNT(*) = 5\n FROM pragma_table_info('tiles_with_hash')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\")\n OR (name = \"tile_hash\" AND type = \"TEXT\"))\n --\n ) as is_valid;\n", - "describe": { - "columns": [ - { - "name": "is_valid", - "ordinal": 0, - "type_info": "Int" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - null - ] - }, - "hash": "3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401" -} diff --git a/martin-mbtiles/.sqlx/query-3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c.json b/martin-mbtiles/.sqlx/query-3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c.json deleted file mode 100644 index 1a8fe4fd0..000000000 --- a/martin-mbtiles/.sqlx/query-3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS sourceDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "3b2930e8d61f31ea1bf32efe340b7766f876ddb9a357a512ab3a37914bea003c" -} diff --git a/martin-mbtiles/.sqlx/query-45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb.json b/martin-mbtiles/.sqlx/query-45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb.json deleted file mode 100644 index 7f4b7b65a..000000000 --- a/martin-mbtiles/.sqlx/query-45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS srcDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "45de99a3628a53940ef80b0e2603c46f61ff92ffbc6ec3bba4860abd60d224cb" -} diff --git a/martin-mbtiles/.sqlx/query-4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3.json b/martin-mbtiles/.sqlx/query-4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3.json new file mode 100644 index 000000000..47392647f --- /dev/null +++ b/martin-mbtiles/.sqlx/query-4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a 'tiles_with_hash' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles_with_hash'\n AND type = 'table'\n --\n ) AND (\n -- 'tiles_with_hash' table's columns and their types are as expected:\n -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).\n -- The order is not important\n SELECT COUNT(*) = 5\n FROM pragma_table_info('tiles_with_hash')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_data' AND type = 'BLOB')\n OR (name = 'tile_hash' AND type = 'TEXT'))\n --\n ) as is_valid;", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "4905d37cd3818e2fe9f65fdd20437901cbe4b6421bac3cf671e86d4b5d8dc0f3" +} diff --git a/martin-mbtiles/.sqlx/query-7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c.json b/martin-mbtiles/.sqlx/query-7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c.json new file mode 100644 index 000000000..2b9d7474d --- /dev/null +++ b/martin-mbtiles/.sqlx/query-7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a 'tiles' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- 'tiles' table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_data' AND type = 'BLOB'))\n --\n ) as is_valid;", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "7341bfc10beb4719811556a57ae8098085994c8fba93e0293359afd43079c50c" +} diff --git a/martin-mbtiles/.sqlx/query-809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13.json b/martin-mbtiles/.sqlx/query-809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13.json new file mode 100644 index 000000000..faf6b51f7 --- /dev/null +++ b/martin-mbtiles/.sqlx/query-809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a 'map' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- 'map' table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE ((name = 'zoom_level' AND type = 'INTEGER')\n OR (name = 'tile_column' AND type = 'INTEGER')\n OR (name = 'tile_row' AND type = 'INTEGER')\n OR (name = 'tile_id' AND type = 'TEXT'))\n --\n ) AND (\n -- Has a 'images' table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- 'images' table's columns and their types are as expected:\n -- 2 columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE ((name = 'tile_id' AND type = 'TEXT')\n OR (name = 'tile_data' AND type = 'BLOB'))\n --\n ) AS is_valid;", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "809e89c3b223e28c6716d405e13ba30fbf018805fe9ca2acd2b2e225183d1f13" +} diff --git a/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json b/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json deleted file mode 100644 index fc0b3c0c2..000000000 --- a/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS newDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b" -} diff --git a/martin-mbtiles/.sqlx/query-b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e.json b/martin-mbtiles/.sqlx/query-b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e.json deleted file mode 100644 index 71fbbc367..000000000 --- a/martin-mbtiles/.sqlx/query-b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS otherDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "b3aaef71d6a26404c3bebcc6ee8ad480aaa224721cd9ddb4ac5859f71a57727e" -} diff --git a/martin-mbtiles/.sqlx/query-d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f.json b/martin-mbtiles/.sqlx/query-d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f.json deleted file mode 100644 index 1d9e5c432..000000000 --- a/martin-mbtiles/.sqlx/query-d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS originalDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "d1d61dfa7c34dafb4588f78e23b2ee47cfc72b56f6ed275a0b0688047405498f" -} diff --git a/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json b/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json deleted file mode 100644 index 5d8f76197..000000000 --- a/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS diffDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c" -} diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index ef6a59111..155792850 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "martin-mbtiles" -version = "0.5.0" +version = "0.6.0" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 4df966ce1..96f7d53b9 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -2,9 +2,7 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use log::{error, LevelFilter}; -use martin_mbtiles::{ - apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions, -}; +use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopier}; #[derive(Parser, PartialEq, Eq, Debug)] #[command( @@ -48,7 +46,7 @@ enum Commands { }, /// Copy tiles from one mbtiles file to another. #[command(name = "copy")] - Copy(TileCopierOptions), + Copy(TileCopier), /// Apply diff file generated from 'copy' command #[command(name = "apply-diff")] ApplyDiff { @@ -165,7 +163,7 @@ mod tests { use clap::error::ErrorKind; use clap::Parser; - use martin_mbtiles::{CopyDuplicateMode, TileCopierOptions}; + use martin_mbtiles::{CopyDuplicateMode, TileCopier}; use crate::Commands::{ApplyDiff, Copy, MetaGetValue, MetaSetValue, Validate}; use crate::{Args, IntegrityCheckType}; @@ -186,7 +184,7 @@ mod tests { Args::parse_from(["mbtiles", "copy", "src_file", "dst_file"]), Args { verbose: false, - command: Copy(TileCopierOptions::new( + command: Copy(TileCopier::new( PathBuf::from("src_file"), PathBuf::from("dst_file") )) @@ -210,7 +208,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .min_zoom(Some(1)) .max_zoom(Some(100)) ) @@ -270,7 +268,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .zoom_levels(vec![1, 3, 7]) ) } @@ -291,7 +289,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .diff_with_file(PathBuf::from("no_file")) ) } @@ -312,7 +310,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .on_duplicate(CopyDuplicateMode::Override) ) } @@ -333,7 +331,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .on_duplicate(CopyDuplicateMode::Ignore) ) } @@ -354,7 +352,7 @@ mod tests { Args { verbose: false, command: Copy( - TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) + TileCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .on_duplicate(CopyDuplicateMode::Abort) ) } diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index 65522b564..c88a8439e 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -10,6 +10,6 @@ mod mbtiles_pool; pub use mbtiles_pool::MbtilesPool; mod tile_copier; -pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions}; +pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopier}; mod mbtiles_queries; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index a3d350dc9..c6f69892b 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -123,6 +123,19 @@ impl Mbtiles { } } + /// Attach this MBTiles file to the given SQLite connection as a given name + pub async fn attach_to(&self, conn: &mut T, name: &str) -> MbtResult<()> + where + for<'e> &'e mut T: SqliteExecutor<'e>, + { + query(&format!("ATTACH DATABASE ? AS {name}")) + .bind(self.filepath()) + .execute(conn) + .await?; + Ok(()) + } + + /// Get a single metadata value from the metadata table pub async fn get_metadata_value(&self, conn: &mut T, key: &str) -> MbtResult> where for<'e> &'e mut T: SqliteExecutor<'e>, @@ -359,6 +372,11 @@ impl Mbtiles { Ok(None) } + pub async fn open_and_detect_type(&self) -> MbtResult { + let mut conn = self.open_with_hashes(true).await?; + self.detect_type(&mut conn).await + } + pub async fn detect_type(&self, conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, @@ -583,7 +601,8 @@ where } pub async fn attach_hash_fn(conn: &mut SqliteConnection) -> MbtResult<()> { - let handle = conn.lock_handle().await?.as_raw_handle().as_ptr(); + let mut handle_lock = conn.lock_handle().await?; + let handle = handle_lock.as_raw_handle().as_ptr(); // Safety: we know that the handle is a SQLite connection is locked and is not used anywhere else. // The registered functions will be dropped when SQLX drops DB connection. let rc = unsafe { sqlite_hashes::rusqlite::Connection::from_handle(handle) }?; @@ -600,25 +619,26 @@ mod tests { use super::*; - async fn open(filepath: &str) -> (SqliteConnection, Mbtiles) { - let mbt = Mbtiles::new(filepath).unwrap(); - let mut conn = SqliteConnection::connect(mbt.filepath()).await.unwrap(); - attach_hash_fn(&mut conn).await.unwrap(); - (conn, mbt) + async fn open(filepath: &str) -> MbtResult<(SqliteConnection, Mbtiles)> { + let mbt = Mbtiles::new(filepath)?; + let mut conn = SqliteConnection::connect(mbt.filepath()).await?; + attach_hash_fn(&mut conn).await?; + Ok((conn, mbt)) } #[actix_rt::test] - async fn mbtiles_meta() { + async fn mbtiles_meta() -> MbtResult<()> { let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"; - let mbt = Mbtiles::new(filepath).unwrap(); + let mbt = Mbtiles::new(filepath)?; assert_eq!(mbt.filepath(), filepath); assert_eq!(mbt.filename(), "geography-class-jpg"); + Ok(()) } #[actix_rt::test] - async fn metadata_jpeg() { - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; - let metadata = mbt.get_metadata(&mut conn).await.unwrap(); + async fn metadata_jpeg() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?; + let metadata = mbt.get_metadata(&mut conn).await?; let tj = metadata.tilejson; assert_eq!(tj.description.unwrap(), "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. "); @@ -630,12 +650,13 @@ mod tests { assert_eq!(tj.version.unwrap(), "1.0.0"); assert_eq!(metadata.id, "geography-class-jpg"); assert_eq!(metadata.tile_info, Format::Jpeg.into()); + Ok(()) } #[actix_rt::test] - async fn metadata_mvt() { - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; - let metadata = mbt.get_metadata(&mut conn).await.unwrap(); + async fn metadata_mvt() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?; + let metadata = mbt.get_metadata(&mut conn).await?; let tj = metadata.tilejson; assert_eq!(tj.maxzoom.unwrap(), 6); @@ -661,41 +682,38 @@ mod tests { TileInfo::new(Format::Mvt, Encoding::Gzip) ); assert_eq!(metadata.layer_type, Some("overlay".to_string())); + Ok(()) } #[actix_rt::test] - async fn metadata_get_key() { - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; - - let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap(); - assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706"); - let res = mbt.get_metadata_value(&mut conn, "name").await.unwrap(); - assert_eq!(res.unwrap(), "Major cities from Natural Earth data"); - let res = mbt.get_metadata_value(&mut conn, "maxzoom").await.unwrap(); - assert_eq!(res.unwrap(), "6"); - let res = mbt.get_metadata_value(&mut conn, "nonexistent_key").await; - assert_eq!(res.unwrap(), None); - let res = mbt.get_metadata_value(&mut conn, "").await; - assert_eq!(res.unwrap(), None); + async fn metadata_get_key() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?; + + let res = mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap(); + assert_eq!(res, "-123.123590,-37.818085,174.763027,59.352706"); + let res = mbt.get_metadata_value(&mut conn, "name").await?.unwrap(); + assert_eq!(res, "Major cities from Natural Earth data"); + let res = mbt.get_metadata_value(&mut conn, "maxzoom").await?.unwrap(); + assert_eq!(res, "6"); + let res = mbt.get_metadata_value(&mut conn, "nonexistent_key").await?; + assert_eq!(res, None); + let res = mbt.get_metadata_value(&mut conn, "").await?; + assert_eq!(res, None); + Ok(()) } #[actix_rt::test] - async fn metadata_set_key() { - let (mut conn, mbt) = open("file:metadata_set_key_mem_db?mode=memory&cache=shared").await; + async fn metadata_set_key() -> MbtResult<()> { + let (mut conn, mbt) = open("file:metadata_set_key_mem_db?mode=memory&cache=shared").await?; query("CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);") .execute(&mut conn) - .await - .unwrap(); + .await?; mbt.set_metadata_value(&mut conn, "bounds", Some("0.0, 0.0, 0.0, 0.0".to_string())) - .await - .unwrap(); + .await?; assert_eq!( - mbt.get_metadata_value(&mut conn, "bounds") - .await - .unwrap() - .unwrap(), + mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap(), "0.0, 0.0, 0.0, 0.0" ); @@ -704,58 +722,53 @@ mod tests { "bounds", Some("-123.123590,-37.818085,174.763027,59.352706".to_string()), ) - .await - .unwrap(); + .await?; assert_eq!( - mbt.get_metadata_value(&mut conn, "bounds") - .await - .unwrap() - .unwrap(), + mbt.get_metadata_value(&mut conn, "bounds").await?.unwrap(), "-123.123590,-37.818085,174.763027,59.352706" ); - mbt.set_metadata_value(&mut conn, "bounds", None) - .await - .unwrap(); - assert_eq!( - mbt.get_metadata_value(&mut conn, "bounds").await.unwrap(), - None - ); + mbt.set_metadata_value(&mut conn, "bounds", None).await?; + assert_eq!(mbt.get_metadata_value(&mut conn, "bounds").await?, None); + + Ok(()) } #[actix_rt::test] - async fn detect_type() { - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await; - let res = mbt.detect_type(&mut conn).await.unwrap(); + async fn detect_type() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?; + let res = mbt.detect_type(&mut conn).await?; assert_eq!(res, MbtType::Flat); - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; - let res = mbt.detect_type(&mut conn).await.unwrap(); + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await?; + let res = mbt.detect_type(&mut conn).await?; assert_eq!(res, MbtType::FlatWithHash); - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await; - let res = mbt.detect_type(&mut conn).await.unwrap(); + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?; + let res = mbt.detect_type(&mut conn).await?; assert_eq!(res, MbtType::Normalized); - let (mut conn, mbt) = open(":memory:").await; + let (mut conn, mbt) = open(":memory:").await?; let res = mbt.detect_type(&mut conn).await; assert!(matches!(res, Err(MbtError::InvalidDataFormat(_)))); + + Ok(()) } #[actix_rt::test] - async fn validate_valid_file() { - let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await; - + async fn validate_valid_file() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await?; mbt.check_integrity(&mut conn, IntegrityCheckType::Quick) - .await - .unwrap(); + .await?; + Ok(()) } #[actix_rt::test] - async fn validate_invalid_file() { + async fn validate_invalid_file() -> MbtResult<()> { let (mut conn, mbt) = - open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await; + open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await?; let result = mbt.check_agg_tiles_hashes(&mut conn).await; assert!(matches!(result, Err(MbtError::AggHashMismatch(..)))); + Ok(()) } } diff --git a/martin-mbtiles/src/mbtiles_queries.rs b/martin-mbtiles/src/mbtiles_queries.rs index 0d04815da..9df2731bf 100644 --- a/martin-mbtiles/src/mbtiles_queries.rs +++ b/martin-mbtiles/src/mbtiles_queries.rs @@ -7,42 +7,41 @@ where for<'e> &'e mut T: SqliteExecutor<'e>, { let sql = query!( - r#"SELECT ( - -- Has a "map" table - SELECT COUNT(*) = 1 - FROM sqlite_master - WHERE name = 'map' - AND type = 'table' - -- - ) AND ( - -- "map" table's columns and their types are as expected: - -- 4 columns (zoom_level, tile_column, tile_row, tile_id). - -- The order is not important - SELECT COUNT(*) = 4 - FROM pragma_table_info('map') - WHERE ((name = "zoom_level" AND type = "INTEGER") - OR (name = "tile_column" AND type = "INTEGER") - OR (name = "tile_row" AND type = "INTEGER") - OR (name = "tile_id" AND type = "TEXT")) - -- - ) AND ( - -- Has a "images" table - SELECT COUNT(*) = 1 - FROM sqlite_master - WHERE name = 'images' - AND type = 'table' - -- - ) AND ( - -- "images" table's columns and their types are as expected: - -- 2 columns (tile_id, tile_data). - -- The order is not important - SELECT COUNT(*) = 2 - FROM pragma_table_info('images') - WHERE ((name = "tile_id" AND type = "TEXT") - OR (name = "tile_data" AND type = "BLOB")) - -- - ) AS is_valid; -"# + "SELECT ( + -- Has a 'map' table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'map' + AND type = 'table' + -- + ) AND ( + -- 'map' table's columns and their types are as expected: + -- 4 columns (zoom_level, tile_column, tile_row, tile_id). + -- The order is not important + SELECT COUNT(*) = 4 + FROM pragma_table_info('map') + WHERE ((name = 'zoom_level' AND type = 'INTEGER') + OR (name = 'tile_column' AND type = 'INTEGER') + OR (name = 'tile_row' AND type = 'INTEGER') + OR (name = 'tile_id' AND type = 'TEXT')) + -- + ) AND ( + -- Has a 'images' table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'images' + AND type = 'table' + -- + ) AND ( + -- 'images' table's columns and their types are as expected: + -- 2 columns (tile_id, tile_data). + -- The order is not important + SELECT COUNT(*) = 2 + FROM pragma_table_info('images') + WHERE ((name = 'tile_id' AND type = 'TEXT') + OR (name = 'tile_data' AND type = 'BLOB')) + -- + ) AS is_valid;" ); Ok(sql @@ -58,27 +57,26 @@ where for<'e> &'e mut T: SqliteExecutor<'e>, { let sql = query!( - r#"SELECT ( - -- Has a "tiles_with_hash" table + "SELECT ( + -- Has a 'tiles_with_hash' table SELECT COUNT(*) = 1 FROM sqlite_master WHERE name = 'tiles_with_hash' - AND type = 'table' + AND type = 'table' -- ) AND ( - -- "tiles_with_hash" table's columns and their types are as expected: + -- 'tiles_with_hash' table's columns and their types are as expected: -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash). -- The order is not important SELECT COUNT(*) = 5 FROM pragma_table_info('tiles_with_hash') - WHERE ((name = "zoom_level" AND type = "INTEGER") - OR (name = "tile_column" AND type = "INTEGER") - OR (name = "tile_row" AND type = "INTEGER") - OR (name = "tile_data" AND type = "BLOB") - OR (name = "tile_hash" AND type = "TEXT")) + WHERE ((name = 'zoom_level' AND type = 'INTEGER') + OR (name = 'tile_column' AND type = 'INTEGER') + OR (name = 'tile_row' AND type = 'INTEGER') + OR (name = 'tile_data' AND type = 'BLOB') + OR (name = 'tile_hash' AND type = 'TEXT')) -- - ) as is_valid; -"# + ) as is_valid;" ); Ok(sql @@ -94,26 +92,25 @@ where for<'e> &'e mut T: SqliteExecutor<'e>, { let sql = query!( - r#"SELECT ( - -- Has a "tiles" table - SELECT COUNT(*) = 1 - FROM sqlite_master - WHERE name = 'tiles' - AND type = 'table' - -- - ) AND ( - -- "tiles" table's columns and their types are as expected: - -- 4 columns (zoom_level, tile_column, tile_row, tile_data). - -- The order is not important - SELECT COUNT(*) = 4 - FROM pragma_table_info('tiles') - WHERE ((name = "zoom_level" AND type = "INTEGER") - OR (name = "tile_column" AND type = "INTEGER") - OR (name = "tile_row" AND type = "INTEGER") - OR (name = "tile_data" AND type = "BLOB")) - -- - ) as is_valid; -"# + "SELECT ( + -- Has a 'tiles' table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'tiles' + AND type = 'table' + -- + ) AND ( + -- 'tiles' table's columns and their types are as expected: + -- 4 columns (zoom_level, tile_column, tile_row, tile_data). + -- The order is not important + SELECT COUNT(*) = 4 + FROM pragma_table_info('tiles') + WHERE ((name = 'zoom_level' AND type = 'INTEGER') + OR (name = 'tile_column' AND type = 'INTEGER') + OR (name = 'tile_row' AND type = 'INTEGER') + OR (name = 'tile_data' AND type = 'BLOB')) + -- + ) as is_valid;" ); Ok(sql @@ -123,3 +120,121 @@ where .unwrap_or_default() == 1) } + +pub async fn create_metadata_table(conn: &mut T) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + query( + "CREATE TABLE IF NOT EXISTS metadata ( + name text NOT NULL PRIMARY KEY, + value text);", + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub async fn create_flat_tables(conn: &mut T) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + query( + "CREATE TABLE IF NOT EXISTS tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub async fn create_flat_with_hash_tables(conn: &mut T) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + query( + "CREATE TABLE IF NOT EXISTS tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + ) + .execute(&mut *conn) + .await?; + + query( + "CREATE VIEW IF NOT EXISTS tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;", + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub async fn create_normalized_tables(conn: &mut T) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + query( + "CREATE TABLE IF NOT EXISTS map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + ) + .execute(&mut *conn) + .await?; + + query( + "CREATE TABLE IF NOT EXISTS images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY);", + ) + .execute(&mut *conn) + .await?; + + query( + "CREATE VIEW IF NOT EXISTS tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id;", + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub async fn create_tiles_with_hash_view(conn: &mut T) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + query( + "CREATE VIEW IF NOT EXISTS tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id", + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index 8e8ced1b5..b7e18e361 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -11,6 +11,10 @@ use sqlx::{query, Connection, Row, SqliteConnection}; use crate::errors::MbtResult; use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; use crate::mbtiles::{attach_hash_fn, MbtType}; +use crate::mbtiles_queries::{ + create_flat_tables, create_flat_with_hash_tables, create_metadata_table, + create_normalized_tables, create_tiles_with_hash_view, +}; use crate::{MbtError, Mbtiles}; #[derive(PartialEq, Eq, Default, Debug, Clone)] @@ -24,7 +28,7 @@ pub enum CopyDuplicateMode { #[derive(Clone, Default, PartialEq, Eq, Debug)] #[cfg_attr(feature = "cli", derive(Args))] -pub struct TileCopierOptions { +pub struct TileCopier { /// MBTiles file to read from src_file: PathBuf, /// MBTiles file to write to @@ -85,13 +89,13 @@ impl clap::builder::TypedValueParser for HashSetValueParser { } #[derive(Clone, Debug)] -struct TileCopier { +struct TileCopierInt { src_mbtiles: Mbtiles, dst_mbtiles: Mbtiles, - options: TileCopierOptions, + options: TileCopier, } -impl TileCopierOptions { +impl TileCopier { #[must_use] pub fn new(src_filepath: PathBuf, dst_filepath: PathBuf) -> Self { Self { @@ -150,13 +154,13 @@ impl TileCopierOptions { } pub async fn run(self) -> MbtResult { - TileCopier::new(self)?.run().await + TileCopierInt::new(self)?.run().await } } -impl TileCopier { - pub fn new(options: TileCopierOptions) -> MbtResult { - Ok(TileCopier { +impl TileCopierInt { + pub fn new(options: TileCopier) -> MbtResult { + Ok(TileCopierInt { src_mbtiles: Mbtiles::new(&options.src_file)?, dst_mbtiles: Mbtiles::new(&options.dst_file)?, options, @@ -164,7 +168,8 @@ impl TileCopier { } pub async fn run(self) -> MbtResult { - let src_type = open_and_detect_type(&self.src_mbtiles).await?; + // src file connection is not needed after this point, as it will be attached to the dst file + let src_type = self.src_mbtiles.open_and_detect_type().await?; let mut conn = SqliteConnection::connect_with( &SqliteConnectOptions::new() @@ -189,7 +194,7 @@ impl TileCopier { return Err(MbtError::NonEmptyTargetFile(self.options.dst_file)); } else { let dst_type = self.dst_mbtiles.detect_type(&mut conn).await?; - attach_source_db(&mut conn, self.src_mbtiles.filepath()).await?; + self.src_mbtiles.attach_to(&mut conn, "sourceDb").await?; dst_type }; @@ -198,11 +203,8 @@ impl TileCopier { let (select_from, query_args) = { let select_from = if let Some(diff_file) = &self.options.diff_with_file { let diff_with_mbtiles = Mbtiles::new(diff_file)?; - let diff_type = open_and_detect_type(&diff_with_mbtiles).await?; - let path = diff_with_mbtiles.filepath(); - query!("ATTACH DATABASE ? AS newDb", path) - .execute(&mut conn) - .await?; + let diff_type = diff_with_mbtiles.open_and_detect_type().await?; + diff_with_mbtiles.attach_to(&mut conn, "newDb").await?; Self::get_select_from_with_diff(dst_type, diff_type) } else { Self::get_select_from(dst_type, src_type).to_string() @@ -213,34 +215,40 @@ impl TileCopier { (format!("{select_from} {options_sql}"), query_args) }; - let handle = conn.lock_handle().await?.as_raw_handle().as_ptr(); - let rusqlite_conn = unsafe { rusqlite::Connection::from_handle(handle) }?; - match dst_type { - Flat => rusqlite_conn.execute( - &format!("INSERT {on_dupl} INTO tiles {select_from} {sql_cond}"), - params_from_iter(query_args), - )?, - FlatWithHash => rusqlite_conn.execute( - &format!("INSERT {on_dupl} INTO tiles_with_hash {select_from} {sql_cond}"), - params_from_iter(query_args), - )?, - Normalized => { - rusqlite_conn.execute( - &format!( - "INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) + { + // Make sure not to execute any other queries while the handle is locked + let mut handle_lock = conn.lock_handle().await?; + let handle = handle_lock.as_raw_handle().as_ptr(); + + // SAFETY: this is safe as long as handle_lock is valid + let rusqlite_conn = unsafe { rusqlite::Connection::from_handle(handle) }?; + match dst_type { + Flat => rusqlite_conn.execute( + &format!("INSERT {on_dupl} INTO tiles {select_from} {sql_cond}"), + params_from_iter(query_args), + )?, + FlatWithHash => rusqlite_conn.execute( + &format!("INSERT {on_dupl} INTO tiles_with_hash {select_from} {sql_cond}"), + params_from_iter(query_args), + )?, + Normalized => { + rusqlite_conn.execute( + &format!( + "INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, hash as tile_id FROM ({select_from} {sql_cond})" - ), - params_from_iter(&query_args), - )?; - rusqlite_conn.execute( - &format!( - "INSERT OR IGNORE INTO images SELECT tile_data, hash FROM ({select_from})" - ), - params_from_iter(query_args), - )? - } - }; + ), + params_from_iter(&query_args), + )?; + rusqlite_conn.execute( + &format!( + "INSERT OR IGNORE INTO images SELECT tile_data, hash FROM ({select_from})" + ), + params_from_iter(query_args), + )? + } + }; + } if !self.options.skip_agg_tiles_hash { self.dst_mbtiles.update_agg_tiles_hash(&mut conn).await?; @@ -258,7 +266,7 @@ impl TileCopier { query!("PRAGMA page_size = 512").execute(&mut *conn).await?; query!("VACUUM").execute(&mut *conn).await?; - attach_source_db(&mut *conn, self.src_mbtiles.filepath()).await?; + self.src_mbtiles.attach_to(&mut *conn, "sourceDb").await?; if src == dst { // DB objects must be created in a specific order: tables, views, triggers, indexes. @@ -281,27 +289,17 @@ impl TileCopier { query(row.get(0)).execute(&mut *conn).await?; } } else { + create_metadata_table(&mut *conn).await?; match dst { - Flat => self.create_flat_tables(&mut *conn).await?, - FlatWithHash => self.create_flat_with_hash_tables(&mut *conn).await?, - Normalized => self.create_normalized_tables(&mut *conn).await?, + Flat => create_flat_tables(&mut *conn).await?, + FlatWithHash => create_flat_with_hash_tables(&mut *conn).await?, + Normalized => create_normalized_tables(&mut *conn).await?, }; }; if dst == Normalized { - query( - "CREATE VIEW tiles_with_hash AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data, - images.tile_id AS tile_hash - FROM map - JOIN images ON images.tile_id = map.tile_id", - ) - .execute(&mut *conn) - .await?; + // Some normalized mbtiles files might not have this view, so even if src == dst, it might not exist + create_tiles_with_hash_view(&mut *conn).await?; } query("INSERT INTO metadata SELECT * FROM sourceDb.metadata") @@ -311,64 +309,13 @@ impl TileCopier { Ok(()) } - async fn create_flat_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { - for statement in &[ - "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", - "CREATE TABLE tiles ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row));", - ] { - query(statement).execute(&mut *conn).await?; - } - Ok(()) - } - - async fn create_flat_with_hash_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { - for statement in &[ - "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", - "CREATE TABLE tiles_with_hash ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - tile_hash text, - PRIMARY KEY(zoom_level, tile_column, tile_row));", - "CREATE VIEW tiles AS - SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;", - ] { - query(statement).execute(&mut *conn).await?; - } - Ok(()) - } - - async fn create_normalized_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { - for statement in &[ - "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", - "CREATE TABLE map ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_id text, - PRIMARY KEY(zoom_level, tile_column, tile_row));", - "CREATE TABLE images (tile_data blob, tile_id text NOT NULL PRIMARY KEY);", - "CREATE VIEW tiles AS - SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data - FROM map - JOIN images ON images.tile_id = map.tile_id;"] { - query(statement).execute(&mut *conn).await?; - } - Ok(()) - } - - fn get_on_duplicate_sql(&self, mbttype: MbtType) -> (String, String) { + /// Returns (ON DUPLICATE SQL, WHERE condition SQL) + fn get_on_duplicate_sql(&self, dst_type: MbtType) -> (String, String) { match &self.options.on_duplicate { CopyDuplicateMode::Override => ("OR REPLACE".to_string(), String::new()), CopyDuplicateMode::Ignore => ("OR IGNORE".to_string(), String::new()), CopyDuplicateMode::Abort => ("OR ABORT".to_string(), { - let (main_table, tile_identifier) = match mbttype { + let (main_table, tile_identifier) = match dst_type { Flat => ("tiles", "tile_data"), FlatWithHash => ("tiles_with_hash", "tile_data"), Normalized => ("map", "tile_id"), @@ -418,12 +365,12 @@ impl TileCopier { fn get_select_from(dst_type: MbtType, src_type: MbtType) -> &'static str { if dst_type == Flat { - "SELECT * FROM sourceDb.tiles WHERE TRUE " + "SELECT * FROM sourceDb.tiles WHERE TRUE" } else { match src_type { - Flat => "SELECT zoom_level, tile_column, tile_row, tile_data, hex(md5(tile_data)) as hash FROM sourceDb.tiles WHERE TRUE ", - FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM sourceDb.tiles_with_hash WHERE TRUE ", - Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id = sourceDb.images.tile_id WHERE TRUE " + Flat => "SELECT zoom_level, tile_column, tile_row, tile_data, hex(md5(tile_data)) as hash FROM sourceDb.tiles WHERE TRUE", + FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM sourceDb.tiles_with_hash WHERE TRUE", + Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id = sourceDb.images.tile_id WHERE TRUE" } } } @@ -459,34 +406,15 @@ impl TileCopier { } } -async fn attach_source_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> { - query!("ATTACH DATABASE ? AS sourceDb", path) - .execute(&mut *conn) - .await?; - Ok(()) -} - -async fn open_and_detect_type(mbtiles: &Mbtiles) -> MbtResult { - let opt = SqliteConnectOptions::new() - .read_only(true) - .filename(mbtiles.filepath()); - let mut conn = SqliteConnection::connect_with(&opt).await?; - mbtiles.detect_type(&mut conn).await -} - pub async fn apply_mbtiles_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> { let src_mbtiles = Mbtiles::new(src_file)?; let diff_mbtiles = Mbtiles::new(diff_file)?; - - let src_type = open_and_detect_type(&src_mbtiles).await?; - let diff_type = open_and_detect_type(&diff_mbtiles).await?; + let diff_type = diff_mbtiles.open_and_detect_type().await?; let mut conn = src_mbtiles.open_with_hashes(false).await?; - let path = diff_mbtiles.filepath(); - query!("ATTACH DATABASE ? AS diffDb", path) - .execute(&mut conn) - .await?; + diff_mbtiles.attach_to(&mut conn, "diffDb").await?; + let src_type = src_mbtiles.detect_type(&mut conn).await?; let select_from = if src_type == Flat { "SELECT zoom_level, tile_column, tile_row, tile_data FROM diffDb.tiles" } else { @@ -534,20 +462,6 @@ mod tests { use super::*; - async fn attach_other_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> { - query!("ATTACH DATABASE ? AS otherDb", path) - .execute(&mut *conn) - .await?; - Ok(()) - } - - async fn attach_src_db(conn: &mut SqliteConnection, path: &str) -> MbtResult<()> { - query!("ATTACH DATABASE ? AS srcDb", path) - .execute(&mut *conn) - .await?; - Ok(()) - } - async fn get_one(conn: &mut SqliteConnection, sql: &str) -> T where for<'r> T: Decode<'r, Sqlite> + Type, @@ -561,15 +475,19 @@ mod tests { dst_type: Option, expected_dst_type: MbtType, ) -> MbtResult<()> { - let mut dst_conn = TileCopierOptions::new(src_filepath.clone(), dst_filepath.clone()) + let mut dst_conn = TileCopier::new(src_filepath.clone(), dst_filepath.clone()) .dst_type(dst_type) .run() .await?; - attach_src_db(&mut dst_conn, src_filepath.to_str().unwrap()).await?; + Mbtiles::new(src_filepath)? + .attach_to(&mut dst_conn, "srcDb") + .await?; assert_eq!( - open_and_detect_type(&Mbtiles::new(dst_filepath)?).await?, + Mbtiles::new(dst_filepath)? + .detect_type(&mut dst_conn) + .await?, expected_dst_type ); @@ -584,7 +502,7 @@ mod tests { } async fn verify_copy_with_zoom_filter( - opts: TileCopierOptions, + opts: TileCopier, expected_zoom_levels: u8, ) -> MbtResult<()> { let mut dst_conn = opts.run().await?; @@ -678,7 +596,7 @@ mod tests { async fn copy_with_min_max_zoom() -> MbtResult<()> { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); - let opt = TileCopierOptions::new(src, dst) + let opt = TileCopier::new(src, dst) .min_zoom(Some(2)) .max_zoom(Some(4)); verify_copy_with_zoom_filter(opt, 3).await @@ -688,7 +606,7 @@ mod tests { async fn copy_with_zoom_levels() -> MbtResult<()> { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); - let opt = TileCopierOptions::new(src, dst) + let opt = TileCopier::new(src, dst) .min_zoom(Some(2)) .max_zoom(Some(4)) .zoom_levels(vec![1, 6]); @@ -703,8 +621,7 @@ mod tests { let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); - let copy_opts = - TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); + let copy_opts = TileCopier::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); let mut dst_conn = copy_opts.run().await?; @@ -752,9 +669,7 @@ mod tests { "file:ignore_dst_type_when_copy_to_existing_mem_db?mode=memory&cache=shared", ); - let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone()) - .run() - .await?; + let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?; verify_copy_all(src_file, dst, Some(Normalized), Flat).await } @@ -765,7 +680,7 @@ mod tests { let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let copy_opts = - TileCopierOptions::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort); + TileCopier::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort); assert!(matches!( copy_opts.run().await.unwrap_err(), @@ -782,16 +697,14 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone()) - .run() - .await?; + let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?; - let mut dst_conn = TileCopierOptions::new(src_file.clone(), dst.clone()) - .run() - .await?; + let mut dst_conn = TileCopier::new(src_file.clone(), dst.clone()).run().await?; // Verify the tiles in the destination file is a superset of the tiles in the source file - attach_other_db(&mut dst_conn, src_file.to_str().unwrap()).await?; + Mbtiles::new(src_file)? + .attach_to(&mut dst_conn, "otherDb") + .await?; assert!( query("SELECT * FROM otherDb.tiles EXCEPT SELECT * FROM tiles;") .fetch_optional(&mut dst_conn) @@ -811,21 +724,19 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = TileCopierOptions::new(dst_file.clone(), dst.clone()) - .run() - .await?; + let _dst_conn = TileCopier::new(dst_file.clone(), dst.clone()).run().await?; - let mut dst_conn = TileCopierOptions::new(src_file.clone(), dst.clone()) + let mut dst_conn = TileCopier::new(src_file.clone(), dst.clone()) .on_duplicate(CopyDuplicateMode::Ignore) .run() .await?; // Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row) - attach_src_db(&mut dst_conn, src_file.to_str().unwrap()).await?; - - let path = dst_file.to_str().unwrap(); - query!("ATTACH DATABASE ? AS originalDb", path) - .execute(&mut dst_conn) + Mbtiles::new(src_file)? + .attach_to(&mut dst_conn, "srcDb") + .await?; + Mbtiles::new(dst_file)? + .attach_to(&mut dst_conn, "originalDb") .await?; // Create a temporary table with all the tiles in the original database and @@ -839,8 +750,7 @@ mod tests { FULL OUTER JOIN srcDb.tiles as t2 ON t1.zoom_level = t2.zoom_level AND t1.tile_column = t2.tile_column AND t1.tile_row = t2.tile_row") .execute(&mut dst_conn) - .await - ?; + .await?; // Ensure all entries in expected_tiles are in tiles and vice versa assert!(query( @@ -861,17 +771,16 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = TileCopier::new(src_file.clone(), src.clone()).run().await?; // Apply diff to the src data in in-memory DB let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/mbtiles/world_cities_modified.mbtiles"; - attach_other_db(&mut src_conn, path).await?; + Mbtiles::new("../tests/fixtures/mbtiles/world_cities_modified.mbtiles")? + .attach_to(&mut src_conn, "otherDb") + .await?; assert!( query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;") @@ -889,17 +798,16 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = TileCopierOptions::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = TileCopier::new(src_file.clone(), src.clone()).run().await?; // Apply diff to the src data in in-memory DB let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles"); apply_mbtiles_diff(src, diff_file).await?; // Verify the data is the same as the file the diff was generated from - let path = "../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"; - attach_other_db(&mut src_conn, path).await?; + Mbtiles::new("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles")? + .attach_to(&mut src_conn, "otherDb") + .await?; assert!( query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")