add encoding=UTF8 pragma, refactoring (#919)

Cargo.toml

@@ -55,6 +55,6 @@ tilejson = "0.3"
 tokio = { version = "1.32.0", features = ["macros"] }
 tokio-postgres-rustls = "0.10"
 
-[profile.dev.package.sqlx-macros]
+[profile.dev.package]
 # See https://github.com/launchbadge/sqlx#compile-time-verification
-opt-level = 3
+sqlx-macros.opt-level = 3

justfile

@@ -134,7 +134,7 @@ test-int: clean-test install-sqlx
 # Run integration tests and save its output as the new expected output
 bless: start clean-test
     rm -rf tests/temp
-    cargo test --features bless-tests
+    cargo test -p martin --features bless-tests
     tests/test.sh
     rm -rf tests/expected
     mv tests/output tests/expected

martin-mbtiles/src/bin/main.rs

@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
 
 use clap::{Parser, Subcommand};
 use log::{error, LevelFilter};
-use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier};
+use martin_mbtiles::{apply_diff, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier};
 
 #[derive(Parser, PartialEq, Eq, Debug)]
 #[command(
@@ -104,14 +104,14 @@ async fn main_int() -> anyhow::Result<()> {
             src_file,
             diff_file,
         } => {
-            apply_mbtiles_diff(src_file, diff_file).await?;
+            apply_diff(src_file, diff_file).await?;
         }
         Commands::Validate {
             file,
             integrity_check,
             update_agg_tiles_hash,
         } => {
-            validate_mbtiles(file.as_path(), integrity_check, update_agg_tiles_hash).await?;
+            validate(file.as_path(), integrity_check, update_agg_tiles_hash).await?;
         }
     }
 
@@ -120,15 +120,15 @@ async fn main_int() -> anyhow::Result<()> {
 
 async fn meta_print_all(file: &Path) -> anyhow::Result<()> {
     let mbt = Mbtiles::new(file)?;
-    let mut conn = mbt.open_with_hashes(true).await?;
+    let mut conn = mbt.open_readonly().await?;
     let metadata = mbt.get_metadata(&mut conn).await?;
     println!("{}", serde_yaml::to_string(&metadata)?);
     Ok(())
 }
 
 async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let mut conn = mbt.open_with_hashes(true).await?;
+    let mut conn = mbt.open_readonly().await?;
     if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? {
         println!("{s}");
     }
@@ -137,17 +137,21 @@ async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
 
 async fn meta_set_value(file: &Path, key: &str, value: Option<String>) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let mut conn = mbt.open_with_hashes(false).await?;
+    let mut conn = mbt.open().await?;
     mbt.set_metadata_value(&mut conn, key, value).await
 }
 
-async fn validate_mbtiles(
+async fn validate(
     file: &Path,
     check_type: IntegrityCheckType,
     update_agg_tiles_hash: bool,
 ) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let mut conn = mbt.open_with_hashes(!update_agg_tiles_hash).await?;
+    let mut conn = if update_agg_tiles_hash {
+        mbt.open().await?
+    } else {
+        mbt.open_readonly().await?
+    };
     mbt.check_integrity(&mut conn, check_type).await?;
     mbt.check_each_tile_hash(&mut conn).await?;
     if update_agg_tiles_hash {

martin-mbtiles/src/copier.rs

@@ -6,14 +6,14 @@ use clap::{builder::ValueParser, error::ErrorKind, Args, ValueEnum};
 use sqlite_hashes::rusqlite;
 use sqlite_hashes::rusqlite::params_from_iter;
 use sqlx::sqlite::SqliteConnectOptions;
-use sqlx::{query, Connection, Row, SqliteConnection};
+use sqlx::{query, Connection, Executor as _, Row, SqliteConnection};
 
 use crate::errors::MbtResult;
 use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized};
 use crate::mbtiles::{attach_hash_fn, MbtType};
 use crate::queries::{
-    create_flat_tables, create_flat_with_hash_tables, create_metadata_table,
-    create_normalized_tables, create_tiles_with_hash_view,
+    create_flat_tables, create_flat_with_hash_tables, create_normalized_tables,
+    create_tiles_with_hash_view,
 };
 use crate::{MbtError, Mbtiles};
 
@@ -187,8 +187,7 @@ impl MbtileCopierInt {
 
         let dst_type = if is_empty {
             let dst_type = self.options.dst_type.unwrap_or(src_type);
-            self.create_new_mbtiles(&mut conn, src_type, dst_type)
-                .await?;
+            self.copy_to_new(&mut conn, src_type, dst_type).await?;
             dst_type
         } else if self.options.diff_with_file.is_some() {
             return Err(MbtError::NonEmptyTargetFile(self.options.dst_file));
@@ -257,39 +256,41 @@ impl MbtileCopierInt {
         Ok(conn)
     }
 
-    async fn create_new_mbtiles(
+    async fn copy_to_new(
         &self,
         conn: &mut SqliteConnection,
         src: MbtType,
         dst: MbtType,
     ) -> MbtResult<()> {
         query!("PRAGMA page_size = 512").execute(&mut *conn).await?;
+        query!("PRAGMA encoding = 'UTF-8'")
+            .execute(&mut *conn)
+            .await?;
         query!("VACUUM").execute(&mut *conn).await?;
 
         self.src_mbtiles.attach_to(&mut *conn, "sourceDb").await?;
 
         if src == dst {
             // DB objects must be created in a specific order: tables, views, triggers, indexes.
-            let sql_objects = query(
-                "SELECT sql
-                 FROM sourceDb.sqlite_schema
-                 WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images', 'tiles_with_hash')
-                     AND type IN ('table', 'view', 'trigger', 'index')
-                 ORDER BY CASE
-                     WHEN type = 'table' THEN 1
-                     WHEN type = 'view' THEN 2
-                     WHEN type = 'trigger' THEN 3
-                     WHEN type = 'index' THEN 4
-                     ELSE 5 END",
-            )
-            .fetch_all(&mut *conn)
-            .await?;
+            let sql_objects = conn
+                .fetch_all(
+                    "SELECT sql
+                     FROM sourceDb.sqlite_schema
+                     WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images', 'tiles_with_hash')
+                         AND type IN ('table', 'view', 'trigger', 'index')
+                     ORDER BY CASE
+                         WHEN type = 'table' THEN 1
+                         WHEN type = 'view' THEN 2
+                         WHEN type = 'trigger' THEN 3
+                         WHEN type = 'index' THEN 4
+                         ELSE 5 END",
+                )
+                .await?;
 
             for row in sql_objects {
                 query(row.get(0)).execute(&mut *conn).await?;
             }
         } else {
-            create_metadata_table(&mut *conn).await?;
             match dst {
                 Flat => create_flat_tables(&mut *conn).await?,
                 FlatWithHash => create_flat_with_hash_tables(&mut *conn).await?,
@@ -302,8 +303,7 @@ impl MbtileCopierInt {
             create_tiles_with_hash_view(&mut *conn).await?;
         }
 
-        query("INSERT INTO metadata SELECT * FROM sourceDb.metadata")
-            .execute(&mut *conn)
+        conn.execute("INSERT INTO metadata SELECT * FROM sourceDb.metadata")
             .await?;
 
         Ok(())
@@ -406,12 +406,12 @@ impl MbtileCopierInt {
     }
 }
 
-pub async fn apply_mbtiles_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> {
+pub async fn apply_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> {
     let src_mbtiles = Mbtiles::new(src_file)?;
     let diff_mbtiles = Mbtiles::new(diff_file)?;
     let diff_type = diff_mbtiles.open_and_detect_type().await?;
 
-    let mut conn = src_mbtiles.open_with_hashes(false).await?;
+    let mut conn = src_mbtiles.open().await?;
     diff_mbtiles.attach_to(&mut conn, "diffDb").await?;
 
     let src_type = src_mbtiles.detect_type(&mut conn).await?;
@@ -491,12 +491,10 @@ mod tests {
             expected_dst_type
         );
 
-        assert!(
-            query("SELECT * FROM srcDb.tiles EXCEPT SELECT * FROM tiles")
-                .fetch_optional(&mut dst_conn)
-                .await?
-                .is_none()
-        );
+        assert!(dst_conn
+            .fetch_optional("SELECT * FROM srcDb.tiles EXCEPT SELECT * FROM tiles")
+            .await?
+            .is_none());
 
         Ok(())
     }
@@ -626,8 +624,8 @@ mod tests {
 
         let mut dst_conn = copy_opts.run().await?;
 
-        assert!(query("SELECT 1 FROM sqlite_schema WHERE name = 'tiles';")
-            .fetch_optional(&mut dst_conn)
+        assert!(dst_conn
+            .fetch_optional("SELECT 1 FROM sqlite_schema WHERE name = 'tiles';")
             .await?
             .is_some());
 
@@ -712,12 +710,10 @@ mod tests {
         Mbtiles::new(src_file)?
             .attach_to(&mut dst_conn, "otherDb")
             .await?;
-        assert!(
-            query("SELECT * FROM otherDb.tiles EXCEPT SELECT * FROM tiles;")
-                .fetch_optional(&mut dst_conn)
-                .await?
-                .is_none()
-        );
+        assert!(dst_conn
+            .fetch_optional("SELECT * FROM otherDb.tiles EXCEPT SELECT * FROM tiles;")
+            .await?
+            .is_none());
 
         Ok(())
     }
@@ -750,15 +746,15 @@ mod tests {
 
         // Create a temporary table with all the tiles in the original database and
         // all the tiles in the source database except for those that conflict with tiles in the original database
-        query("CREATE TEMP TABLE expected_tiles AS
+        dst_conn.execute(
+            "CREATE TEMP TABLE expected_tiles AS
                    SELECT COALESCE(t1.zoom_level, t2.zoom_level) as zoom_level,
                           COALESCE(t1.tile_column, t2.zoom_level) as tile_column,
                           COALESCE(t1.tile_row, t2.tile_row) as tile_row,
                           COALESCE(t1.tile_data, t2.tile_data) as tile_data
                    FROM originalDb.tiles as t1
                    FULL OUTER JOIN srcDb.tiles as t2
                        ON t1.zoom_level = t2.zoom_level AND t1.tile_column = t2.tile_column AND t1.tile_row = t2.tile_row")
-            .execute(&mut dst_conn)
             .await?;
 
         // Ensure all entries in expected_tiles are in tiles and vice versa
@@ -786,19 +782,17 @@ mod tests {
 
         // Apply diff to the src data in in-memory DB
         let diff_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles");
-        apply_mbtiles_diff(src, diff_file).await?;
+        apply_diff(src, diff_file).await?;
 
         // Verify the data is the same as the file the diff was generated from
         Mbtiles::new("../tests/fixtures/mbtiles/world_cities_modified.mbtiles")?
             .attach_to(&mut src_conn, "otherDb")
             .await?;
 
-        assert!(
-            query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
-                .fetch_optional(&mut src_conn)
-                .await?
-                .is_none()
-        );
+        assert!(src_conn
+            .fetch_optional("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
+            .await?
+            .is_none());
 
         Ok(())
     }
@@ -815,19 +809,17 @@ mod tests {
 
         // Apply diff to the src data in in-memory DB
         let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles");
-        apply_mbtiles_diff(src, diff_file).await?;
+        apply_diff(src, diff_file).await?;
 
         // Verify the data is the same as the file the diff was generated from
         Mbtiles::new("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles")?
             .attach_to(&mut src_conn, "otherDb")
             .await?;
 
-        assert!(
-            query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
-                .fetch_optional(&mut src_conn)
-                .await?
-                .is_none()
-        );
+        assert!(src_conn
+            .fetch_optional("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;")
+            .await?
+            .is_none());
 
         Ok(())
     }

martin-mbtiles/src/lib.rs

@@ -10,6 +10,6 @@ mod pool;
 pub use pool::MbtilesPool;
 
 mod copier;
-pub use copier::{apply_mbtiles_diff, CopyDuplicateMode, MbtilesCopier};
+pub use copier::{apply_diff, CopyDuplicateMode, MbtilesCopier};
 
 mod queries;

martin-mbtiles/src/mbtiles.rs

@@ -93,11 +93,27 @@ impl Mbtiles {
         })
     }
 
-    pub async fn open_with_hashes(&self, readonly: bool) -> MbtResult<SqliteConnection> {
+    pub async fn open(&self) -> MbtResult<SqliteConnection> {
+        let opt = SqliteConnectOptions::new().filename(self.filepath());
+        Self::open_int(&opt).await
+    }
+
+    pub async fn open_or_new(&self) -> MbtResult<SqliteConnection> {
+        let opt = SqliteConnectOptions::new()
+            .filename(self.filepath())
+            .create_if_missing(true);
+        Self::open_int(&opt).await
+    }
+
+    pub async fn open_readonly(&self) -> MbtResult<SqliteConnection> {
         let opt = SqliteConnectOptions::new()
             .filename(self.filepath())
-            .read_only(readonly);
-        let mut conn = SqliteConnection::connect_with(&opt).await?;
+            .read_only(true);
+        Self::open_int(&opt).await
+    }
+
+    async fn open_int(opt: &SqliteConnectOptions) -> Result<SqliteConnection, MbtError> {
+        let mut conn = SqliteConnection::connect_with(opt).await?;
         attach_hash_fn(&mut conn).await?;
         Ok(conn)
     }
@@ -373,7 +389,7 @@ impl Mbtiles {
     }
 
     pub async fn open_and_detect_type(&self) -> MbtResult<MbtType> {
-        let mut conn = self.open_with_hashes(true).await?;
+        let mut conn = self.open_readonly().await?;
         self.detect_type(&mut conn).await
     }
 
@@ -615,6 +631,7 @@ mod tests {
     use std::collections::HashMap;
 
     use martin_tile_utils::Encoding;
+    use sqlx::Executor as _;
     use tilejson::VectorLayer;
 
     use super::*;
@@ -706,8 +723,7 @@ mod tests {
     async fn metadata_set_key() -> MbtResult<()> {
         let (mut conn, mbt) = open("file:metadata_set_key_mem_db?mode=memory&cache=shared").await?;
 
-        query("CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);")
-            .execute(&mut conn)
+        conn.execute("CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);")
             .await?;
 
         mbt.set_metadata_value(&mut conn, "bounds", Some("0.0, 0.0, 0.0, 0.0".to_string()))