From 6247a219ec003a53e001b942a9cf98a3847d86c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20HUBERT=20/=20PALO-IT?= Date: Tue, 11 Jul 2023 11:39:06 +0200 Subject: [PATCH 1/4] fix tar file pointer --- .github/workflows/ci.yml | 2 +- mithril-client/src/utils/unpacker.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d539e696290..9d37a6807bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,7 @@ jobs: fail-fast: false matrix: era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }} - run_id: [1,2,3] + run_id: [1,2,3,4,5,6,7,8,9,10] steps: - name: Checkout sources uses: actions/checkout@v3 diff --git a/mithril-client/src/utils/unpacker.rs b/mithril-client/src/utils/unpacker.rs index f8169332e95..f2234cc1d3d 100644 --- a/mithril-client/src/utils/unpacker.rs +++ b/mithril-client/src/utils/unpacker.rs @@ -1,5 +1,6 @@ use std::{ fs::{create_dir_all, remove_dir, File}, + io::{Seek, SeekFrom}, path::{Path, PathBuf}, }; @@ -90,12 +91,15 @@ impl SnapshotUnpacker { /// Unpack the snapshot pointed at the given filepath into the given directory. pub async fn unpack_snapshot(&self, filepath: &Path, unpack_dir: &Path) -> StdResult<()> { - let snapshot_file_tar_gz = + let mut snapshot_file_tar_gz = File::open(filepath).map_err(|e| SnapshotUnpackerError::UnpackFailed { filepath: filepath.to_owned(), dirpath: unpack_dir.to_owned(), error: e.into(), })?; + // Try to force the file read to start at 0. We need to determine if + // this fixes instabilities in CI. + snapshot_file_tar_gz.seek(SeekFrom::Start(0))?; let snapshot_file_tar = GzDecoder::new(snapshot_file_tar_gz); let mut snapshot_archive = Archive::new(snapshot_file_tar); snapshot_archive From e3f41edc41a7eb3fa6f85d3f4ff99669c46e5c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20HUBERT=20/=20PALO-IT?= Date: Tue, 11 Jul 2023 12:05:48 +0200 Subject: [PATCH 2/4] update compression lib version --- mithril-client/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml index a7a1a7ec0db..d892b54e2ee 100644 --- a/mithril-client/Cargo.toml +++ b/mithril-client/Cargo.toml @@ -17,7 +17,7 @@ clap = { version = "4.0", features = ["derive", "env"] } cli-table = "0.4" config = "0.13.1" directories = "5.0.1" -flate2 = "1.0.23" +flate2 = "1.0.26" fs2 = "0.4.3" futures = "0.3" hex = "0.4.3" From ef058aa9e554b8b449cbe3cb4c4d1ee1170f6d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20HUBERT=20/=20PALO-IT?= Date: Tue, 11 Jul 2023 14:34:34 +0200 Subject: [PATCH 3/4] add extension to downloaded snapshots --- .github/workflows/ci.yml | 2 +- .../src/aggregator_client/snapshot_client.rs | 2 +- mithril-client/src/services/snapshot.rs | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d37a6807bd..d539e696290 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,7 @@ jobs: fail-fast: false matrix: era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }} - run_id: [1,2,3,4,5,6,7,8,9,10] + run_id: [1,2,3] steps: - name: Checkout sources uses: actions/checkout@v3 diff --git a/mithril-client/src/aggregator_client/snapshot_client.rs b/mithril-client/src/aggregator_client/snapshot_client.rs index 8b2a5764b29..b620d77150d 100644 --- a/mithril-client/src/aggregator_client/snapshot_client.rs +++ b/mithril-client/src/aggregator_client/snapshot_client.rs @@ -67,7 +67,7 @@ impl SnapshotClient { ) -> StdResult { let filepath = PathBuf::new() .join(download_dir) - .join(format!("snapshot-{}", snapshot.digest)); + .join(format!("snapshot-{}.tar.gz", snapshot.digest)); for url in snapshot.locations.as_slice() { if self.http_client.probe(url).await.is_ok() { diff --git a/mithril-client/src/services/snapshot.rs b/mithril-client/src/services/snapshot.rs index 1015ad6254d..e9d26e04e7b 100644 --- a/mithril-client/src/services/snapshot.rs +++ b/mithril-client/src/services/snapshot.rs @@ -500,7 +500,11 @@ mod tests { let (_, verifier) = setup_genesis(); let genesis_verification_key = verifier.to_verification_key(); - build_dummy_snapshot("digest-10", "1234567890".repeat(124).as_str(), &test_path); + build_dummy_snapshot( + "digest-10.tar.gz", + "1234567890".repeat(124).as_str(), + &test_path, + ); let filepath = snapshot_service .download( &snapshot, @@ -551,7 +555,11 @@ mod tests { let (_, verifier) = setup_genesis(); let genesis_verification_key = verifier.to_verification_key(); - build_dummy_snapshot("digest-10", "1234567890".repeat(124).as_str(), &test_path); + build_dummy_snapshot( + "digest-10.tar.gz", + "1234567890".repeat(124).as_str(), + &test_path, + ); let err = snapshot_service .download( &signed_entity, @@ -577,7 +585,7 @@ mod tests { "Expected a SnapshotServiceError when snapshot can not be verified. Got {err:?}: '{err}'" ); } - let filepath = test_path.join("snapshot-digest-10"); + let filepath = test_path.join("snapshot-digest-10.tar.gz"); assert!(filepath.exists()); let unpack_dir = filepath .parent() From da54c0fc232999c51f3744402d67794a3b24d4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20HUBERT=20/=20PALO-IT?= Date: Tue, 11 Jul 2023 15:34:07 +0200 Subject: [PATCH 4/4] update version --- Cargo.lock | 2 +- mithril-client/Cargo.toml | 2 +- mithril-client/src/utils/unpacker.rs | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38dc0c6ef8d..3e0490d05db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2132,7 +2132,7 @@ dependencies = [ [[package]] name = "mithril-client" -version = "0.3.17" +version = "0.3.18" dependencies = [ "async-recursion", "async-trait", diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml index d892b54e2ee..e1f8cce4bbf 100644 --- a/mithril-client/Cargo.toml +++ b/mithril-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client" -version = "0.3.17" +version = "0.3.18" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-client/src/utils/unpacker.rs b/mithril-client/src/utils/unpacker.rs index f2234cc1d3d..67bf099e8e9 100644 --- a/mithril-client/src/utils/unpacker.rs +++ b/mithril-client/src/utils/unpacker.rs @@ -97,8 +97,9 @@ impl SnapshotUnpacker { dirpath: unpack_dir.to_owned(), error: e.into(), })?; - // Try to force the file read to start at 0. We need to determine if - // this fixes instabilities in CI. + // Try to force the file read to start at 0. + // This seems to fix a crash when the unpacker tries to iterate + // over archive content. snapshot_file_tar_gz.seek(SeekFrom::Start(0))?; let snapshot_file_tar = GzDecoder::new(snapshot_file_tar_gz); let mut snapshot_archive = Archive::new(snapshot_file_tar);