From f300504073a64ab3f62054ec3f09bd7ba6a0e805 Mon Sep 17 00:00:00 2001 From: antheas Date: Fri, 28 Jun 2024 17:12:33 +0200 Subject: [PATCH 01/16] add contentmeta option with determinism --- lib/src/chunking.rs | 20 ++++++++------- lib/src/cli.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/lib/src/chunking.rs b/lib/src/chunking.rs index 4b047e740..17c470b5b 100644 --- a/lib/src/chunking.rs +++ b/lib/src/chunking.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use std::borrow::{Borrow, Cow}; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Write; use std::hash::{Hash, Hasher}; use std::num::NonZeroU32; @@ -53,9 +53,9 @@ pub(crate) struct Chunk { pub struct ObjectSourceMetaSized { /// The original metadata #[serde(flatten)] - meta: ObjectSourceMeta, + pub meta: ObjectSourceMeta, /// Total size of associated objects - size: u64, + pub size: u64, } impl Hash for ObjectSourceMetaSized { @@ -89,7 +89,7 @@ impl ObjectMetaSized { let map = meta.map; let mut set = meta.set; // Maps content id -> total size of associated objects - let mut sizes = HashMap::<&str, u64>::new(); + let mut sizes = BTreeMap::<&str, u64>::new(); // Populate two mappings above, iterating over the object -> contentid mapping for (checksum, contentid) in map.iter() { let finfo = repo.query_file(checksum, cancellable)?.0; @@ -308,7 +308,7 @@ impl Chunking { } // Reverses `contentmeta.map` i.e. contentid -> Vec - let mut rmap = HashMap::>::new(); + let mut rmap = BTreeMap::>::new(); for (checksum, contentid) in meta.map.iter() { rmap.entry(Rc::clone(contentid)).or_default().push(checksum); } @@ -577,12 +577,12 @@ fn basic_packing_with_prior_build<'a>( let mut curr_build = curr_build?; // View the packages as unordered sets for lookups and differencing - let prev_pkgs_set: HashSet = curr_build + let prev_pkgs_set: BTreeSet = curr_build .iter() .flat_map(|v| v.iter().cloned()) .filter(|name| !name.is_empty()) .collect(); - let curr_pkgs_set: HashSet = components + let curr_pkgs_set: BTreeSet = components .iter() .map(|pkg| pkg.meta.name.to_string()) .collect(); @@ -597,13 +597,13 @@ fn basic_packing_with_prior_build<'a>( } // Handle removed packages - let removed: HashSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect(); + let removed: BTreeSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect(); for bin in curr_build.iter_mut() { bin.retain(|pkg| !removed.contains(pkg)); } // Handle updated packages - let mut name_to_component: HashMap = HashMap::new(); + let mut name_to_component: BTreeMap = BTreeMap::new(); for component in components.iter() { name_to_component .entry(component.meta.name.to_string()) @@ -821,6 +821,8 @@ mod test { } fn create_manifest(prev_expected_structure: Vec>) -> oci_spec::image::ImageManifest { + use std::collections::HashMap; + let mut p = prev_expected_structure .iter() .map(|b| { diff --git a/lib/src/cli.rs b/lib/src/cli.rs index c9f91cb43..9b22b14a7 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -19,16 +19,20 @@ use std::collections::BTreeMap; use std::ffi::OsString; use std::fs::File; use std::io::{BufReader, BufWriter, Write}; +use std::num::NonZeroU32; use std::path::PathBuf; use std::process::Command; use tokio::sync::mpsc::Receiver; +use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized}; use crate::commit::container_commit; use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport}; use crate::container::{self as ostree_container, ManifestDiff}; use crate::container::{Config, ImageReference, OstreeImageReference}; +use crate::objectsource::ObjectSourceMeta; use crate::sysroot::SysrootLock; use ostree_container::store::{ImageImporter, PrepareResult}; +use serde::{Deserialize, Serialize}; /// Parse an [`OstreeImageReference`] from a CLI arguemnt. pub fn parse_imgref(s: &str) -> Result { @@ -165,6 +169,10 @@ pub(crate) enum ContainerOpts { /// Compress at the fastest level (e.g. gzip level 1) #[clap(long)] compression_fast: bool, + + /// Path to a JSON-formatted content meta object. + #[clap(long)] + contentmeta: Option, }, /// Perform build-time checking and canonicalization. @@ -699,6 +707,15 @@ async fn container_import( Ok(()) } +/// Grouping of metadata about an object. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RawMeta { + /// ContentId to layer annotation + pub layers: BTreeMap, + /// OSTree hash to layer ContentId + pub mapping: BTreeMap, +} + /// Export a container image with an encapsulated ostree commit. #[allow(clippy::too_many_arguments)] async fn container_export( @@ -712,6 +729,7 @@ async fn container_export( container_config: Option, cmd: Option>, compression_fast: bool, + contentmeta: Option, ) -> Result<()> { let config = Config { labels: Some(labels), @@ -722,12 +740,54 @@ async fn container_export( } else { None }; + let contentmeta = if let Some(contentmeta) = contentmeta { + let raw: Option = + serde_json::from_reader(File::open(contentmeta).map(BufReader::new)?)?; + if let Some(raw) = raw { + Some(ObjectMetaSized { + map: raw + .mapping + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + sizes: raw + .layers + .into_iter() + .map(|(k, v)| ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: k.clone().into(), + name: v.into(), + srcid: k.clone().into(), + change_frequency: if k == "unpackaged" { std::u32::MAX } else { 1 }, + change_time_offset: 1, + }, + size: 1, + }) + .collect(), + }) + } else { + anyhow::bail!("Content metadata must be a JSON object") + } + } else { + None + }; + + // Use enough layers so that each package ends in its own layer + // while respecting the layer ordering. + let max_layers = if let Some(contentmeta) = &contentmeta { + NonZeroU32::new(contentmeta.sizes.len().try_into().unwrap()) + } else { + None + }; + let opts = crate::container::ExportOpts { copy_meta_keys, copy_meta_opt_keys, container_config, authfile, skip_compression: compression_fast, // TODO rename this in the struct at the next semver break + contentmeta: contentmeta.as_ref(), + max_layers, ..Default::default() }; let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?; @@ -958,6 +1018,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + contentmeta, } => { let labels: Result> = labels .into_iter() @@ -980,6 +1041,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + contentmeta, ) .await } From ad24d8ca122063882ca1ec45ac84e17c70b5a8d3 Mon Sep 17 00:00:00 2001 From: antheas Date: Sun, 30 Jun 2024 16:50:25 +0200 Subject: [PATCH 02/16] add label support to top level --- lib/src/cli.rs | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 9b22b14a7..1d661cbb7 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -710,6 +710,8 @@ async fn container_import( /// Grouping of metadata about an object. #[derive(Debug, Default, Serialize, Deserialize)] pub struct RawMeta { + /// Top level labels, to be prefixed to the ones with --label + pub labels: Option>, /// ContentId to layer annotation pub layers: BTreeMap, /// OSTree hash to layer ContentId @@ -731,20 +733,19 @@ async fn container_export( compression_fast: bool, contentmeta: Option, ) -> Result<()> { - let config = Config { - labels: Some(labels), - cmd, - }; let container_config = if let Some(container_config) = container_config { serde_json::from_reader(File::open(container_config).map(BufReader::new)?)? } else { None }; - let contentmeta = if let Some(contentmeta) = contentmeta { + + let mut contentmeta_data = None; + let mut labels = labels.clone(); + if let Some(contentmeta) = contentmeta { let raw: Option = serde_json::from_reader(File::open(contentmeta).map(BufReader::new)?)?; if let Some(raw) = raw { - Some(ObjectMetaSized { + contentmeta_data = Some(ObjectMetaSized { map: raw .mapping .into_iter() @@ -764,29 +765,36 @@ async fn container_export( size: 1, }) .collect(), - }) + }); + // Allow --label to override labels from the content metadata + if let Some(raw_labels) = raw.labels { + labels = raw_labels.into_iter().chain(labels.into_iter()).collect(); + }; } else { anyhow::bail!("Content metadata must be a JSON object") } - } else { - None - }; + } // Use enough layers so that each package ends in its own layer // while respecting the layer ordering. - let max_layers = if let Some(contentmeta) = &contentmeta { - NonZeroU32::new(contentmeta.sizes.len().try_into().unwrap()) + let max_layers = if let Some(contentmeta_data) = &contentmeta_data { + NonZeroU32::new(contentmeta_data.sizes.len().try_into().unwrap()) } else { None }; + let config = Config { + labels: Some(labels), + cmd, + }; + let opts = crate::container::ExportOpts { copy_meta_keys, copy_meta_opt_keys, container_config, authfile, skip_compression: compression_fast, // TODO rename this in the struct at the next semver break - contentmeta: contentmeta.as_ref(), + contentmeta: contentmeta_data.as_ref(), max_layers, ..Default::default() }; From 050916ee52a4e36422b750bbe97ed69c854ac9b2 Mon Sep 17 00:00:00 2001 From: antheas Date: Sun, 30 Jun 2024 18:00:21 +0200 Subject: [PATCH 03/16] force ordering in internal mappings (does not seem to make a difference/remove) --- lib/Cargo.toml | 1 + lib/src/chunking.rs | 3 ++- lib/src/cli.rs | 5 +++-- lib/src/objectsource.rs | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index eb931d9ad..8218647c8 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -50,6 +50,7 @@ tokio-util = { features = ["io-util"], version = "0.7" } tokio-stream = { features = ["sync"], version = "0.1.8" } tracing = "0.1" zstd = { version = "0.13.1", features = ["pkg-config"] } +indexmap = { version = "2.2.6", features = ["serde"] } indoc = { version = "2", optional = true } xshell = { version = "0.2", optional = true } diff --git a/lib/src/chunking.rs b/lib/src/chunking.rs index 17c470b5b..abe5f60cd 100644 --- a/lib/src/chunking.rs +++ b/lib/src/chunking.rs @@ -19,6 +19,7 @@ use camino::Utf8PathBuf; use containers_image_proxy::oci_spec; use gvariant::aligned_bytes::TryAsAligned; use gvariant::{Marker, Structure}; +use indexmap::IndexMap; use ostree::{gio, glib}; use serde::{Deserialize, Serialize}; @@ -308,7 +309,7 @@ impl Chunking { } // Reverses `contentmeta.map` i.e. contentid -> Vec - let mut rmap = BTreeMap::>::new(); + let mut rmap = IndexMap::>::new(); for (checksum, contentid) in meta.map.iter() { rmap.entry(Rc::clone(contentid)).or_default().push(checksum); } diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 1d661cbb7..c55a4b119 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -12,6 +12,7 @@ use cap_std_ext::cap_std; use cap_std_ext::prelude::CapStdExtDirExt; use clap::{Parser, Subcommand}; use fn_error_context::context; +use indexmap::IndexMap; use io_lifetimes::AsFd; use ostree::{gio, glib}; use std::borrow::Cow; @@ -713,9 +714,9 @@ pub struct RawMeta { /// Top level labels, to be prefixed to the ones with --label pub labels: Option>, /// ContentId to layer annotation - pub layers: BTreeMap, + pub layers: IndexMap, /// OSTree hash to layer ContentId - pub mapping: BTreeMap, + pub mapping: IndexMap, } /// Export a container image with an encapsulated ostree commit. diff --git a/lib/src/objectsource.rs b/lib/src/objectsource.rs index 64a3eb137..95f8cf4af 100644 --- a/lib/src/objectsource.rs +++ b/lib/src/objectsource.rs @@ -3,7 +3,8 @@ //! This is used to help split up containers into distinct layers. use std::borrow::Borrow; -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; +use indexmap::IndexMap; use std::hash::Hash; use std::rc::Rc; @@ -78,7 +79,7 @@ impl Borrow for ObjectSourceMeta { pub type ObjectMetaSet = HashSet; /// Maps from an ostree content object digest to the `ContentSet` key. -pub type ObjectMetaMap = BTreeMap; +pub type ObjectMetaMap = IndexMap; /// Grouping of metadata about an object. #[derive(Debug, Default)] From 92b12be7838131e6e590120958bff579f2c317f5 Mon Sep 17 00:00:00 2001 From: antheas Date: Mon, 1 Jul 2024 16:49:43 +0200 Subject: [PATCH 04/16] fix layer concatenation bug --- lib/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index c55a4b119..9b73ad579 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -779,7 +779,7 @@ async fn container_export( // Use enough layers so that each package ends in its own layer // while respecting the layer ordering. let max_layers = if let Some(contentmeta_data) = &contentmeta_data { - NonZeroU32::new(contentmeta_data.sizes.len().try_into().unwrap()) + NonZeroU32::new((contentmeta_data.sizes.len() + 1).try_into().unwrap()) } else { None }; From 8c94e0546450e5f53d8ffd364100d1d517fbff83 Mon Sep 17 00:00:00 2001 From: antheas Date: Mon, 1 Jul 2024 16:52:09 +0200 Subject: [PATCH 05/16] add annotation setting support --- Cargo.toml | 3 +++ lib/src/container/encapsulate.rs | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0441b93cc..4f16aaf9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,6 @@ debug = true codegen-units = 1 inherits = "release" lto = "yes" + +[patch.crates-io] +oci-spec = { git = "https://github.com/antheas/oci-spec-rs.git" } \ No newline at end of file diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index 3f055f4fa..50d3c18a9 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -189,13 +189,13 @@ fn build_oci( imgcfg.set_created(Some( commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string(), )); - let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default); + let mut labels = HashMap::new(); commit_meta_to_labels( &commit_meta, opts.copy_meta_keys.iter().map(|k| k.as_str()), opts.copy_meta_opt_keys.iter().map(|k| k.as_str()), - labels, + &mut labels, )?; let mut manifest = ocidir::new_empty_manifest().build().unwrap(); @@ -244,7 +244,7 @@ fn build_oci( writer, &mut manifest, &mut imgcfg, - labels, + &mut labels, chunking, &opts, &description, @@ -261,6 +261,8 @@ fn build_oci( ctrcfg.set_cmd(Some(cmd.clone())); } + imgcfg.set_annotations(Some(labels.clone())); + ctrcfg.labels_mut().get_or_insert_with(Default::default).extend(labels); imgcfg.set_config(Some(ctrcfg)); let ctrcfg = writer.write_config(imgcfg)?; manifest.set_config(ctrcfg); From ff6a6daaaad3578e9b05f8d71074d7e6a218dbdc Mon Sep 17 00:00:00 2001 From: antheas Date: Mon, 1 Jul 2024 17:09:31 +0200 Subject: [PATCH 06/16] add created timestamp support --- lib/src/cli.rs | 6 ++++++ lib/src/container/encapsulate.rs | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 9b73ad579..fa64a3213 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -711,6 +711,8 @@ async fn container_import( /// Grouping of metadata about an object. #[derive(Debug, Default, Serialize, Deserialize)] pub struct RawMeta { + /// When the image was created. Sync it with the io.container.image.created label. + pub created: Option, /// Top level labels, to be prefixed to the ones with --label pub labels: Option>, /// ContentId to layer annotation @@ -741,11 +743,14 @@ async fn container_export( }; let mut contentmeta_data = None; + let mut created = None; let mut labels = labels.clone(); if let Some(contentmeta) = contentmeta { let raw: Option = serde_json::from_reader(File::open(contentmeta).map(BufReader::new)?)?; if let Some(raw) = raw { + created = raw.created; + contentmeta_data = Some(ObjectMetaSized { map: raw .mapping @@ -797,6 +802,7 @@ async fn container_export( skip_compression: compression_fast, // TODO rename this in the struct at the next semver break contentmeta: contentmeta_data.as_ref(), max_layers, + created, ..Default::default() }; let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?; diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index 50d3c18a9..1357a424c 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -186,9 +186,12 @@ fn build_oci( let mut ctrcfg = opts.container_config.clone().unwrap_or_default(); let mut imgcfg = oci_image::ImageConfiguration::default(); - imgcfg.set_created(Some( - commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string(), - )); + + let created_at = opts + .created + .clone() + .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string()); + imgcfg.set_created(Some(created_at)); let mut labels = HashMap::new(); commit_meta_to_labels( @@ -262,7 +265,10 @@ fn build_oci( } imgcfg.set_annotations(Some(labels.clone())); - ctrcfg.labels_mut().get_or_insert_with(Default::default).extend(labels); + ctrcfg + .labels_mut() + .get_or_insert_with(Default::default) + .extend(labels); imgcfg.set_config(Some(ctrcfg)); let ctrcfg = writer.write_config(imgcfg)?; manifest.set_config(ctrcfg); @@ -377,6 +383,8 @@ pub struct ExportOpts<'m, 'o> { /// Metadata mapping between objects and their owning component/package; /// used to optimize packing. pub contentmeta: Option<&'o ObjectMetaSized>, + /// Sets the created tag in the image manifest. + pub created: Option, } impl<'m, 'o> ExportOpts<'m, 'o> { From 33b2606aaaf840fa21c409445253634cdeb4f674 Mon Sep 17 00:00:00 2001 From: antheas Date: Mon, 1 Jul 2024 18:30:33 +0200 Subject: [PATCH 07/16] Revert "add annotation setting support" This reverts commit 8c94e0546450e5f53d8ffd364100d1d517fbff83. --- Cargo.toml | 3 --- lib/src/container/encapsulate.rs | 11 +++-------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f16aaf9e..0441b93cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,3 @@ debug = true codegen-units = 1 inherits = "release" lto = "yes" - -[patch.crates-io] -oci-spec = { git = "https://github.com/antheas/oci-spec-rs.git" } \ No newline at end of file diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index 1357a424c..e87cb2623 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -192,13 +192,13 @@ fn build_oci( .clone() .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string()); imgcfg.set_created(Some(created_at)); - let mut labels = HashMap::new(); + let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default); commit_meta_to_labels( &commit_meta, opts.copy_meta_keys.iter().map(|k| k.as_str()), opts.copy_meta_opt_keys.iter().map(|k| k.as_str()), - &mut labels, + labels, )?; let mut manifest = ocidir::new_empty_manifest().build().unwrap(); @@ -247,7 +247,7 @@ fn build_oci( writer, &mut manifest, &mut imgcfg, - &mut labels, + labels, chunking, &opts, &description, @@ -264,11 +264,6 @@ fn build_oci( ctrcfg.set_cmd(Some(cmd.clone())); } - imgcfg.set_annotations(Some(labels.clone())); - ctrcfg - .labels_mut() - .get_or_insert_with(Default::default) - .extend(labels); imgcfg.set_config(Some(ctrcfg)); let ctrcfg = writer.write_config(imgcfg)?; manifest.set_config(ctrcfg); From 05f13f8bfe97a29161909c1e9fd537cfd25fce75 Mon Sep 17 00:00:00 2001 From: antheas Date: Mon, 1 Jul 2024 18:40:54 +0200 Subject: [PATCH 08/16] add annotation support --- lib/src/container/encapsulate.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index e87cb2623..e7964fcf2 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -192,13 +192,13 @@ fn build_oci( .clone() .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string()); imgcfg.set_created(Some(created_at)); - let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default); + let mut labels = HashMap::new(); commit_meta_to_labels( &commit_meta, opts.copy_meta_keys.iter().map(|k| k.as_str()), opts.copy_meta_opt_keys.iter().map(|k| k.as_str()), - labels, + &mut labels, )?; let mut manifest = ocidir::new_empty_manifest().build().unwrap(); @@ -247,7 +247,7 @@ fn build_oci( writer, &mut manifest, &mut imgcfg, - labels, + &mut labels, chunking, &opts, &description, @@ -264,9 +264,14 @@ fn build_oci( ctrcfg.set_cmd(Some(cmd.clone())); } + ctrcfg + .labels_mut() + .get_or_insert_with(Default::default) + .extend(labels.clone()); imgcfg.set_config(Some(ctrcfg)); let ctrcfg = writer.write_config(imgcfg)?; manifest.set_config(ctrcfg); + manifest.set_annotations(Some(labels)); let platform = oci_image::Platform::default(); if let Some(tag) = tag { writer.insert_manifest(manifest, Some(tag), platform)?; From 1061e935696462c0a06aa0f5b8952f6e15658512 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Thu, 1 Aug 2024 16:02:24 +0300 Subject: [PATCH 09/16] use ocidir fork to fix newlines --- lib/Cargo.toml | 2 +- lib/src/container/store.rs | 2 +- lib/src/container/update_detachedmeta.rs | 2 +- lib/src/integrationtest.rs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8218647c8..f322734b8 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -36,7 +36,7 @@ once_cell = "1.9" libc = "0.2.92" libsystemd = "0.7.0" openssl = "0.10.33" -ocidir = "0.1.0" +ocidir = { version = "0.2.0", git = "https://github.com/hhd-dev/ocidir-rs" } pin-project = "1.0" regex = "1.5.4" rustix = { version = "0.38", features = ["fs", "process"] } diff --git a/lib/src/container/store.rs b/lib/src/container/store.rs index 49d2ca659..ae4ea21b0 100644 --- a/lib/src/container/store.rs +++ b/lib/src/container/store.rs @@ -1310,7 +1310,7 @@ pub(crate) fn export_to_oci( let compression = opts.skip_compression.then_some(Compression::none()); for (i, layer) in remaining_layers.iter().enumerate() { let layer_ref = &ref_for_layer(layer)?; - let mut target_blob = dest_oci.create_raw_layer(compression)?; + let mut target_blob = dest_oci.create_gzip_layer(compression)?; // Sadly the libarchive stuff isn't exposed via Rust due to type unsafety, // so we'll just fork off the CLI. let repo_dfd = repo.dfd_borrow(); diff --git a/lib/src/container/update_detachedmeta.rs b/lib/src/container/update_detachedmeta.rs index 50576ed4f..8993680b8 100644 --- a/lib/src/container/update_detachedmeta.rs +++ b/lib/src/container/update_detachedmeta.rs @@ -74,7 +74,7 @@ pub async fn update_detached_metadata( // Create tar streams for source and destination let src_layer = BufReader::new(tempsrc.read_blob(commit_layer)?); let mut src_layer = flate2::read::GzDecoder::new(src_layer); - let mut out_layer = BufWriter::new(tempsrc.create_raw_layer(None)?); + let mut out_layer = BufWriter::new(tempsrc.create_gzip_layer(None)?); // Process the tar stream and inject our new detached metadata crate::tar::update_detached_metadata( diff --git a/lib/src/integrationtest.rs b/lib/src/integrationtest.rs index e7397dd42..3b3e82e00 100644 --- a/lib/src/integrationtest.rs +++ b/lib/src/integrationtest.rs @@ -11,7 +11,7 @@ use containers_image_proxy::oci_spec; use fn_error_context::context; use gio::prelude::*; use oci_spec::image as oci_image; -use ocidir::RawLayerWriter; +use ocidir::GzipLayerWriter; use ostree::gio; use xshell::cmd; @@ -58,7 +58,7 @@ pub fn generate_derived_oci_from_tar( tag: Option<&str>, ) -> Result<()> where - F: FnOnce(&mut RawLayerWriter) -> Result<()>, + F: FnOnce(&mut GzipLayerWriter) -> Result<()>, { let src = src.as_ref(); let src = Dir::open_ambient_dir(src, cap_std::ambient_authority())?; @@ -67,7 +67,7 @@ where let mut manifest = src.read_manifest()?; let mut config: oci_spec::image::ImageConfiguration = src.read_json_blob(manifest.config())?; - let mut bw = src.create_raw_layer(None)?; + let mut bw = src.create_gzip_layer(None)?; f(&mut bw)?; let new_layer = bw.complete()?; From e855c51a0cafeb1d816b2b7f78ae477ee48e224b Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Tue, 27 Aug 2024 23:28:23 +0200 Subject: [PATCH 10/16] remove option<> from rawmeta creation --- lib/src/cli.rs | 63 ++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index fa64a3213..80f74aa04 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -746,39 +746,36 @@ async fn container_export( let mut created = None; let mut labels = labels.clone(); if let Some(contentmeta) = contentmeta { - let raw: Option = - serde_json::from_reader(File::open(contentmeta).map(BufReader::new)?)?; - if let Some(raw) = raw { - created = raw.created; - - contentmeta_data = Some(ObjectMetaSized { - map: raw - .mapping - .into_iter() - .map(|(k, v)| (k.into(), v.into())) - .collect(), - sizes: raw - .layers - .into_iter() - .map(|(k, v)| ObjectSourceMetaSized { - meta: ObjectSourceMeta { - identifier: k.clone().into(), - name: v.into(), - srcid: k.clone().into(), - change_frequency: if k == "unpackaged" { std::u32::MAX } else { 1 }, - change_time_offset: 1, - }, - size: 1, - }) - .collect(), - }); - // Allow --label to override labels from the content metadata - if let Some(raw_labels) = raw.labels { - labels = raw_labels.into_iter().chain(labels.into_iter()).collect(); - }; - } else { - anyhow::bail!("Content metadata must be a JSON object") - } + let buf = File::open(contentmeta).map(BufReader::new); + let raw: RawMeta = serde_json::from_reader(buf?)?; + + created = raw.created; + contentmeta_data = Some(ObjectMetaSized { + map: raw + .mapping + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + sizes: raw + .layers + .into_iter() + .map(|(k, v)| ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: k.clone().into(), + name: v.into(), + srcid: k.clone().into(), + change_frequency: if k == "unpackaged" { std::u32::MAX } else { 1 }, + change_time_offset: 1, + }, + size: 1, + }) + .collect(), + }); + + // Allow --label to override labels from the content metadata + if let Some(raw_labels) = raw.labels { + labels = raw_labels.into_iter().chain(labels.into_iter()).collect(); + }; } // Use enough layers so that each package ends in its own layer From dbef2afe7e5c52f795688e08cf36364ae5822080 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Tue, 27 Aug 2024 23:39:56 +0200 Subject: [PATCH 11/16] improve contentmeta docs and fix label nit --- lib/src/cli.rs | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 80f74aa04..9fae90039 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -711,14 +711,28 @@ async fn container_import( /// Grouping of metadata about an object. #[derive(Debug, Default, Serialize, Deserialize)] pub struct RawMeta { - /// When the image was created. Sync it with the io.container.image.created label. + /// The metadata format version. Should be set to 1. + pub version: Option, + /// The image creation timestamp. Format is YYYY-MM-DDTHH:MM:SSZ. + /// Should be synced with the label io.container.image.created. pub created: Option, /// Top level labels, to be prefixed to the ones with --label + /// Applied to both the outer config annotations and the inner config labels. pub labels: Option>, - /// ContentId to layer annotation + /// The output layers ordered. Provided as an ordered mapping of a unique + /// machine readable strings to a human readable name (e.g., the layer contents). + /// The human-readable name is placed in a layer annotation. pub layers: IndexMap, - /// OSTree hash to layer ContentId + /// The layer contents. The key is an ostree hash and the value is the + /// machine readable string of the layer the hash belongs to. + /// WARNING: needs to contain all ostree hashes in the input commit. pub mapping: IndexMap, + /// Whether the mapping is ordered. If true, the output tar stream of the + /// layers will reflect the order of the hashes in the mapping. + /// Otherwise, a deterministic ordering will be used regardless of mapping + /// order. Potentially useful for optimizing zstd:chunked compression. + /// WARNING: not currently supported. + pub ordered: Option, } /// Export a container image with an encapsulated ostree commit. @@ -749,6 +763,18 @@ async fn container_export( let buf = File::open(contentmeta).map(BufReader::new); let raw: RawMeta = serde_json::from_reader(buf?)?; + // Check future variables are set correctly + if let Some(version) = raw.version { + if version != 1 { + return Err(anyhow::anyhow!("Unsupported metadata version: {}", version)); + } + } + if let Some(ordered) = raw.ordered { + if ordered { + return Err(anyhow::anyhow!("Ordered mapping not currently supported.")); + } + } + created = raw.created; contentmeta_data = Some(ObjectMetaSized { map: raw @@ -772,10 +798,8 @@ async fn container_export( .collect(), }); - // Allow --label to override labels from the content metadata - if let Some(raw_labels) = raw.labels { - labels = raw_labels.into_iter().chain(labels.into_iter()).collect(); - }; + // Merge --label args to the labels from the metadata + labels.extend(raw.labels.into_iter().flatten()); } // Use enough layers so that each package ends in its own layer From f3162f90c221574abba498e686494da4f723ff25 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Fri, 30 Aug 2024 21:40:23 +0200 Subject: [PATCH 12/16] Revert "use ocidir fork to fix newlines" This reverts commit 1061e935696462c0a06aa0f5b8952f6e15658512. --- lib/Cargo.toml | 2 +- lib/src/container/store.rs | 2 +- lib/src/container/update_detachedmeta.rs | 2 +- lib/src/integrationtest.rs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f322734b8..8218647c8 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -36,7 +36,7 @@ once_cell = "1.9" libc = "0.2.92" libsystemd = "0.7.0" openssl = "0.10.33" -ocidir = { version = "0.2.0", git = "https://github.com/hhd-dev/ocidir-rs" } +ocidir = "0.1.0" pin-project = "1.0" regex = "1.5.4" rustix = { version = "0.38", features = ["fs", "process"] } diff --git a/lib/src/container/store.rs b/lib/src/container/store.rs index ae4ea21b0..49d2ca659 100644 --- a/lib/src/container/store.rs +++ b/lib/src/container/store.rs @@ -1310,7 +1310,7 @@ pub(crate) fn export_to_oci( let compression = opts.skip_compression.then_some(Compression::none()); for (i, layer) in remaining_layers.iter().enumerate() { let layer_ref = &ref_for_layer(layer)?; - let mut target_blob = dest_oci.create_gzip_layer(compression)?; + let mut target_blob = dest_oci.create_raw_layer(compression)?; // Sadly the libarchive stuff isn't exposed via Rust due to type unsafety, // so we'll just fork off the CLI. let repo_dfd = repo.dfd_borrow(); diff --git a/lib/src/container/update_detachedmeta.rs b/lib/src/container/update_detachedmeta.rs index 8993680b8..50576ed4f 100644 --- a/lib/src/container/update_detachedmeta.rs +++ b/lib/src/container/update_detachedmeta.rs @@ -74,7 +74,7 @@ pub async fn update_detached_metadata( // Create tar streams for source and destination let src_layer = BufReader::new(tempsrc.read_blob(commit_layer)?); let mut src_layer = flate2::read::GzDecoder::new(src_layer); - let mut out_layer = BufWriter::new(tempsrc.create_gzip_layer(None)?); + let mut out_layer = BufWriter::new(tempsrc.create_raw_layer(None)?); // Process the tar stream and inject our new detached metadata crate::tar::update_detached_metadata( diff --git a/lib/src/integrationtest.rs b/lib/src/integrationtest.rs index 3b3e82e00..e7397dd42 100644 --- a/lib/src/integrationtest.rs +++ b/lib/src/integrationtest.rs @@ -11,7 +11,7 @@ use containers_image_proxy::oci_spec; use fn_error_context::context; use gio::prelude::*; use oci_spec::image as oci_image; -use ocidir::GzipLayerWriter; +use ocidir::RawLayerWriter; use ostree::gio; use xshell::cmd; @@ -58,7 +58,7 @@ pub fn generate_derived_oci_from_tar( tag: Option<&str>, ) -> Result<()> where - F: FnOnce(&mut GzipLayerWriter) -> Result<()>, + F: FnOnce(&mut RawLayerWriter) -> Result<()>, { let src = src.as_ref(); let src = Dir::open_ambient_dir(src, cap_std::ambient_authority())?; @@ -67,7 +67,7 @@ where let mut manifest = src.read_manifest()?; let mut config: oci_spec::image::ImageConfiguration = src.read_json_blob(manifest.config())?; - let mut bw = src.create_gzip_layer(None)?; + let mut bw = src.create_raw_layer(None)?; f(&mut bw)?; let new_layer = bw.complete()?; From cb33dadbc26b8a824e0dabaa4876e164efc5139a Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Fri, 30 Aug 2024 22:03:19 +0200 Subject: [PATCH 13/16] fix formatting --- lib/src/objectsource.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/objectsource.rs b/lib/src/objectsource.rs index 95f8cf4af..f32c56ead 100644 --- a/lib/src/objectsource.rs +++ b/lib/src/objectsource.rs @@ -2,9 +2,9 @@ //! //! This is used to help split up containers into distinct layers. +use indexmap::IndexMap; use std::borrow::Borrow; use std::collections::HashSet; -use indexmap::IndexMap; use std::hash::Hash; use std::rc::Rc; From 3ea0f76b0f046abc4bcb8b62c509b35fdc0e62a8 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Fri, 30 Aug 2024 22:11:59 +0200 Subject: [PATCH 14/16] make version field mandatory --- lib/src/cli.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 9fae90039..6bbfcdf2b 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -712,7 +712,7 @@ async fn container_import( #[derive(Debug, Default, Serialize, Deserialize)] pub struct RawMeta { /// The metadata format version. Should be set to 1. - pub version: Option, + pub version: u32, /// The image creation timestamp. Format is YYYY-MM-DDTHH:MM:SSZ. /// Should be synced with the label io.container.image.created. pub created: Option, @@ -764,10 +764,13 @@ async fn container_export( let raw: RawMeta = serde_json::from_reader(buf?)?; // Check future variables are set correctly - if let Some(version) = raw.version { - if version != 1 { - return Err(anyhow::anyhow!("Unsupported metadata version: {}", version)); - } + let supported_version = 1; + if raw.version != supported_version { + return Err(anyhow::anyhow!( + "Unsupported metadata version: {}. Currently supported: {}", + raw.version, + supported_version + )); } if let Some(ordered) = raw.ordered { if ordered { From 862c1ecf10d598c52ff149d33dcf34f3c119a5a3 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Fri, 30 Aug 2024 22:21:07 +0200 Subject: [PATCH 15/16] fix fixture using the wrong entry type --- lib/src/fixture.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/fixture.rs b/lib/src/fixture.rs index 769e8bf82..7a2ba9211 100644 --- a/lib/src/fixture.rs +++ b/lib/src/fixture.rs @@ -302,7 +302,7 @@ fn build_mapping_recurse( dir: &gio::File, ret: &mut ObjectMeta, ) -> Result<()> { - use std::collections::btree_map::Entry; + use indexmap::map::Entry; let cancellable = gio::Cancellable::NONE; let e = dir.enumerate_children( "standard::name,standard::type", From 8fda0491465f5e56b63be7f52cdf6ae58a5aab38 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Fri, 30 Aug 2024 22:22:05 +0200 Subject: [PATCH 16/16] lower indexmap version to fix c9s --- lib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8218647c8..042230464 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -50,7 +50,7 @@ tokio-util = { features = ["io-util"], version = "0.7" } tokio-stream = { features = ["sync"], version = "0.1.8" } tracing = "0.1" zstd = { version = "0.13.1", features = ["pkg-config"] } -indexmap = { version = "2.2.6", features = ["serde"] } +indexmap = { version = "2.2.2", features = ["serde"] } indoc = { version = "2", optional = true } xshell = { version = "0.2", optional = true }