Update agg_tiles_hash docs and minor bug (#901)

* Do not open the same mbtiles file more than once at the same time:
reuse the (unsafe) sqlite handle to create a `RusqliteConnection`
instead.
* The copying should set `agg_tiles_hash` in all cases because now it
uses the always available `tiles` table/view.
* a few minor cleanups and renames related to that

.github/workflows/ci.yml

@@ -506,7 +506,7 @@ jobs:
           elif [[ "${{ matrix.target }}" == "debian-x86_64" ]]; then
             mv debian-x86_64.deb ../${{ matrix.name }}
           else
-            tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}          
+            tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
           fi
       - name: Generate SHA-256 (MacOS)
         if: matrix.sha == 'true'

docs/src/tools.md

@@ -69,47 +69,64 @@ If the `.mbtiles` file is of `flat_with_hash` or `normalized` type, then verify
 mbtiles validate src_file.mbtiles
 ```
 
+## Content Validation
+The original [MBTiles specification](https://github.com/mapbox/mbtiles-spec#readme) does not provide any guarantees for the content of the tile data in MBTiles. This tool adds a few additional conventions to ensure that the content of the tile data is valid.
+
+A typical Normalized schema generated by tools like [tilelive-copy](https://github.com/mapbox/TileLive#bintilelive-copy) use MD5 hash in the `tile_id` column. The Martin's `mbtiles` tool can use this hash to verify the content of each tile. We also define a new `flat-with-hash` schema that stores the hash and tile data in the same table. This schema is more efficient than the `normalized` schema when data has no duplicate tiles (see below). Per tile validation is not available for `flat` schema.
+
+Per-tile validation will catch individual invalid tiles, but it will not detect overall datastore corruption (e.g. missing tiles or tiles that shouldn't exist, or tiles with incorrect z/x/y values).  For that, Martin `mbtiles` tool defines a new metadata value called `agg_tiles_hash`. The value is computed by hashing  `cast(zoom_level AS text), cast(tile_column AS text), cast(tile_row AS text), cast(tile_data as blob)` combined for all rows in the `tiles` table/view, ordered by z,x,y. In case there are no rows or all are NULL, the hash value of an empty string is used.
+
+The `mbtiles` tool will compute `agg_tiles_hash` value when copying or validating mbtiles files.
+
 ## Supported Schema
-The `mbtiles` tool supports three different kinds of schema for `tiles` data in `.mbtiles` files:
-
-- `flat`:
-    ```
-    CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);
-    CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);
-    ```
-- `flat-with-hash`:
-    ```
-    CREATE TABLE tiles_with_hash (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, tile_hash text);
-    CREATE UNIQUE INDEX tiles_with_hash_index on tiles_with_hash (zoom_level, tile_column, tile_row);
-    CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;
-    ```
-- `normalized`:
-    ```
-    CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT);
-    CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row);
-    CREATE TABLE images (tile_data blob, tile_id text);
-    CREATE UNIQUE INDEX images_id ON images (tile_id);
-    CREATE VIEW tiles AS
-      SELECT
-          map.zoom_level AS zoom_level,
-          map.tile_column AS tile_column,
-          map.tile_row AS tile_row,
-          images.tile_data AS tile_data
-      FROM map
-      JOIN images ON images.tile_id = map.tile_id;
-    ```
-  Optionally, `.mbtiles` files with `normalized` schema can include a `tiles_with_hash` view:
-  ```
-  CREATE VIEW tiles_with_hash AS
-      SELECT
-          map.zoom_level AS zoom_level,
-          map.tile_column AS tile_column,
-          map.tile_row AS tile_row,
-          images.tile_data AS tile_data,
-          images.tile_id AS tile_hash
-      FROM map
-      JOIN images ON images.tile_id = map.tile_id;
-  ```
-  **__Note:__** All `normalized` files created by the `mbtiles` tool will contain this view.
-
-For more general spec information, see [here](https://github.com/mapbox/mbtiles-spec#readme).
+The `mbtiles` tool supports three different kinds of schema for `tiles` data in `.mbtiles` files. See also the original [specification](https://github.com/mapbox/mbtiles-spec#readme).
+
+#### flat
+```sql, ignore
+CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);
+CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);
+```
+
+#### flat-with-hash
+```sql, ignore
+CREATE TABLE tiles_with_hash (
+  zoom_level integer NOT NULL,
+  tile_column integer NOT NULL,
+  tile_row integer NOT NULL,
+  tile_data blob,
+  tile_hash text);
+CREATE UNIQUE INDEX tiles_with_hash_index on tiles_with_hash (zoom_level, tile_column, tile_row);
+CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;
+```
+
+#### normalized
+```sql, ignore
+CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT);
+CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row);
+CREATE TABLE images (tile_data blob, tile_id text);
+CREATE UNIQUE INDEX images_id ON images (tile_id);
+CREATE VIEW tiles AS
+  SELECT
+      map.zoom_level AS zoom_level,
+      map.tile_column AS tile_column,
+      map.tile_row AS tile_row,
+      images.tile_data AS tile_data
+  FROM map
+  JOIN images ON images.tile_id = map.tile_id;
+```
+
+Optionally, `.mbtiles` files with `normalized` schema can include a `tiles_with_hash` view:
+
+```sql, ignore
+CREATE VIEW tiles_with_hash AS
+  SELECT
+      map.zoom_level AS zoom_level,
+      map.tile_column AS tile_column,
+      map.tile_row AS tile_row,
+      images.tile_data AS tile_data,
+      images.tile_id AS tile_hash
+  FROM map
+  JOIN images ON images.tile_id = map.tile_id;
+```
+
+**__Note:__** All `normalized` files created by the `mbtiles` tool will contain this view.

martin-mbtiles/src/bin/main.rs

@@ -2,9 +2,7 @@ use std::path::{Path, PathBuf};
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use martin_mbtiles::{
-    apply_mbtiles_diff, copy_mbtiles_file, IntegrityCheckType, Mbtiles, TileCopierOptions,
-};
+use martin_mbtiles::{apply_mbtiles_diff, IntegrityCheckType, Mbtiles, TileCopierOptions};
 use sqlx::sqlite::SqliteConnectOptions;
 use sqlx::{Connection, SqliteConnection};
 
@@ -67,7 +65,7 @@ enum Commands {
         /// Value to specify the extent of the SQLite integrity check performed
         #[arg(long, value_enum, default_value_t=IntegrityCheckType::default())]
         integrity_check: IntegrityCheckType,
-        /// Generate a hash of the tile data hashes and store under the 'agg_tiles_hash' key in metadata
+        /// Update `agg_tiles_hash` metadata value instead of using it to validate if the entire tile store is valid.
         #[arg(long)]
         update_agg_tiles_hash: bool,
     },
@@ -88,7 +86,7 @@ async fn main() -> Result<()> {
             meta_set_value(file.as_path(), &key, value).await?;
         }
         Commands::Copy(opts) => {
-            copy_mbtiles_file(opts).await?;
+            opts.run().await?;
         }
         Commands::ApplyDiff {
             src_file,
@@ -148,7 +146,7 @@ async fn validate_mbtiles(
     if update_agg_tiles_hash {
         mbt.update_agg_tiles_hash(&mut conn).await?;
     } else {
-        mbt.check_agg_tile_hashes(&mut conn).await?;
+        mbt.check_agg_tiles_hashes(&mut conn).await?;
     }
     Ok(())
 }

martin-mbtiles/src/lib.rs

@@ -9,6 +9,4 @@ mod tile_copier;
 pub use errors::MbtError;
 pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata};
 pub use mbtiles_pool::MbtilesPool;
-pub use tile_copier::{
-    apply_mbtiles_diff, copy_mbtiles_file, CopyDuplicateMode, TileCopierOptions,
-};
+pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions};