Fix mbtiles validation, CI, and logging (#903)

justfile

@@ -110,6 +110,7 @@ test-int: clean-test install-sqlx
 
 # Run integration tests and save its output as the new expected output
 bless: start clean-test
+    rm -rf tests/temp
     cargo test --features bless-tests
     tests/test.sh
     rm -rf tests/expected
@@ -216,7 +217,7 @@ git-pre-push: stop start
 # Update sqlite database schema.
 prepare-sqlite: install-sqlx
     mkdir -p martin-mbtiles/.sqlx
-    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests
+    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/mbtiles/world_cities.mbtiles -- --lib --tests
 
 # Install SQLX cli if not already installed.
 [private]

martin-mbtiles/Cargo.toml

@@ -12,7 +12,7 @@ rust-version.workspace = true
 [features]
 # TODO: Disable "cli" feature in default builds
 default = ["cli", "native-tls"]
-cli = ["dep:anyhow", "dep:clap", "dep:tokio"]
+cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:tokio"]
 # One of the following two must be used
 native-tls = ["sqlx/runtime-tokio-native-tls"]
 rustls = ["sqlx/runtime-tokio-rustls"]
@@ -30,6 +30,7 @@ tilejson.workspace = true
 # Bin dependencies
 anyhow = { workspace = true, optional = true }
 clap = { workspace = true, optional = true }
+env_logger = { workspace = true, optional = true }
 serde_yaml.workspace = true
 sqlite-hashes.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }

martin-mbtiles/src/bin/main.rs

@@ -1,11 +1,10 @@
 use std::path::{Path, PathBuf};
 
 use clap::{Parser, Subcommand};
+use log::{error, LevelFilter};
 use martin_mbtiles::{
     apply_mbtiles_diff, IntegrityCheckType, MbtResult, Mbtiles, TileCopierOptions,
 };
-use sqlx::sqlite::SqliteConnectOptions;
-use sqlx::{Connection, SqliteConnection};
 
 #[derive(Parser, PartialEq, Eq, Debug)]
 #[command(
@@ -73,9 +72,23 @@ enum Commands {
 }
 
 #[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let args = Args::parse();
+async fn main() {
+    env_logger::builder()
+        .filter_level(LevelFilter::Info)
+        .format_indent(None)
+        .format_module_path(false)
+        .format_target(false)
+        .format_timestamp(None)
+        .init();
 
+    if let Err(err) = main_int().await {
+        error!("{err}");
+        std::process::exit(1);
+    }
+}
+
+async fn main_int() -> anyhow::Result<()> {
+    let args = Args::parse();
     match args.command {
         Commands::MetaAll { file } => {
             meta_print_all(file.as_path()).await?;
@@ -109,17 +122,15 @@ async fn main() -> anyhow::Result<()> {
 
 async fn meta_print_all(file: &Path) -> anyhow::Result<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(true).await?;
     let metadata = mbt.get_metadata(&mut conn).await?;
     println!("{}", serde_yaml::to_string(&metadata)?);
     Ok(())
 }
 
 async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(true).await?;
     if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? {
         println!("{s}");
     }
@@ -128,8 +139,7 @@ async fn meta_get_value(file: &Path, key: &str) -> MbtResult<()> {
 
 async fn meta_set_value(file: &Path, key: &str, value: Option<String>) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(false).await?;
     mbt.set_metadata_value(&mut conn, key, value).await
 }
 
@@ -139,8 +149,7 @@ async fn validate_mbtiles(
     update_agg_tiles_hash: bool,
 ) -> MbtResult<()> {
     let mbt = Mbtiles::new(file)?;
-    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
-    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    let mut conn = mbt.open_with_hashes(!update_agg_tiles_hash).await?;
     mbt.check_integrity(&mut conn, check_type).await?;
     mbt.check_each_tile_hash(&mut conn).await?;
     if update_agg_tiles_hash {

martin-mbtiles/src/mbtiles.rs

@@ -435,7 +435,9 @@ impl Mbtiles {
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
+        let filepath = self.filepath();
         if integrity_check == IntegrityCheckType::Off {
+            info!("Skipping integrity check for {filepath}");
             return Ok(());
         }
 
@@ -452,30 +454,32 @@ impl Mbtiles {
 
         if result.len() > 1
             || result.get(0).ok_or(FailedIntegrityCheck(
-                self.filepath().to_string(),
+                filepath.to_string(),
                 vec!["SQLite could not perform integrity check".to_string()],
             ))? != "ok"
         {
             return Err(FailedIntegrityCheck(self.filepath().to_string(), result));
         }
 
+        info!("{integrity_check:?} integrity check passed for {filepath}");
         Ok(())
     }
 
     pub async fn check_agg_tiles_hashes<T>(&self, conn: &mut T) -> MbtResult<()>
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
+        let filepath = self.filepath();
         let Some(stored) = self.get_agg_tiles_hash(&mut *conn).await? else {
-            return Err(AggHashValueNotFound(self.filepath().to_string()));
+            return Err(AggHashValueNotFound(filepath.to_string()));
         };
-
         let computed = calc_agg_tiles_hash(&mut *conn).await?;
         if stored != computed {
-            let file = self.filepath().to_string();
+            let file = filepath.to_string();
             return Err(AggHashMismatch(computed, stored, file));
         }
 
+        info!("The agg_tiles_hashes={computed} has been verified for {filepath}");
         Ok(())
     }
 
@@ -486,23 +490,15 @@ impl Mbtiles {
     {
         let old_hash = self.get_agg_tiles_hash(&mut *conn).await?;
         let hash = calc_agg_tiles_hash(&mut *conn).await?;
+        let path = self.filepath();
         if old_hash.as_ref() == Some(&hash) {
-            info!(
-                "agg_tiles_hash is already set to the correct value `{hash}` in {}",
-                self.filepath()
-            );
+            info!("agg_tiles_hash is already set to the correct value `{hash}` in {path}");
             Ok(())
         } else {
             if let Some(old_hash) = old_hash {
-                info!(
-                    "Updating agg_tiles_hash from {old_hash} to {hash} in {}",
-                    self.filepath()
-                );
+                info!("Updating agg_tiles_hash from {old_hash} to {hash} in {path}");
             } else {
-                info!(
-                    "Initializing agg_tiles_hash to {hash} in {}",
-                    self.filepath()
-                );
+                info!("Creating new metadata value agg_tiles_hash = {hash} in {path}");
             }
             self.set_metadata_value(&mut *conn, "agg_tiles_hash", Some(hash))
                 .await
@@ -550,7 +546,10 @@ impl Mbtiles {
                     v.get(0),
                     v.get(1),
                 ))
-            })
+            })?;
+
+        info!("All tile hashes are valid for {}", self.filepath());
+        Ok(())
     }
 }
 
@@ -610,15 +609,15 @@ mod tests {
 
     #[actix_rt::test]
     async fn mbtiles_meta() {
-        let filepath = "../tests/fixtures/files/geography-class-jpg.mbtiles";
+        let filepath = "../tests/fixtures/mbtiles/geography-class-jpg.mbtiles";
         let mbt = Mbtiles::new(filepath).unwrap();
         assert_eq!(mbt.filepath(), filepath);
         assert_eq!(mbt.filename(), "geography-class-jpg");
     }
 
     #[actix_rt::test]
     async fn metadata_jpeg() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
         let metadata = mbt.get_metadata(&mut conn).await.unwrap();
         let tj = metadata.tilejson;
 
@@ -635,7 +634,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn metadata_mvt() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
         let metadata = mbt.get_metadata(&mut conn).await.unwrap();
         let tj = metadata.tilejson;
 
@@ -666,7 +665,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn metadata_get_key() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
 
         let res = mbt.get_metadata_value(&mut conn, "bounds").await.unwrap();
         assert_eq!(res.unwrap(), "-123.123590,-37.818085,174.763027,59.352706");
@@ -726,15 +725,15 @@ mod tests {
 
     #[actix_rt::test]
     async fn detect_type() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::Flat);
 
-        let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::FlatWithHash);
 
-        let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await;
         let res = mbt.detect_type(&mut conn).await.unwrap();
         assert_eq!(res, MbtType::Normalized);
 
@@ -745,7 +744,7 @@ mod tests {
 
     #[actix_rt::test]
     async fn validate_valid_file() {
-        let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await;
+        let (mut conn, mbt) = open("../tests/fixtures/mbtiles/zoomed_world_cities.mbtiles").await;
 
         mbt.check_integrity(&mut conn, IntegrityCheckType::Quick)
             .await
@@ -755,7 +754,7 @@ mod tests {
     #[actix_rt::test]
     async fn validate_invalid_file() {
         let (mut conn, mbt) =
-            open("../tests/fixtures/files/invalid/invalid_zoomed_world_cities.mbtiles").await;
+            open("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles").await;
         let result = mbt.check_agg_tiles_hashes(&mut conn).await;
         assert!(matches!(result, Err(MbtError::AggHashMismatch(..))));
     }