From 979b93a24765ab129c6f1414388a9c7565fe3d80 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 6 Jul 2023 07:00:49 -0400 Subject: [PATCH] deploy: Add an API to prune undeployed images This is part of fixing https://github.com/coreos/rpm-ostree/issues/4391 but is also in the general theme of making things less "stateful". A huge huge mess today is `rpm-ostree rebase` and `bootc switch` both have `--retain` options which keep the previous image. But really what we want is to use the deployments as source-of-truth; that way if e.g. an admin pins a deployment, it automatically pins the image too. And this will help strongly align with the bootc direction in reconciling to desired state. --- ci/priv-integration.sh | 7 ++++- lib/src/cli.rs | 37 ++++++++++++++++++++++++++ lib/src/container/deploy.rs | 52 ++++++++++++++++++++++++++++++++++++- lib/src/container/mod.rs | 8 +++--- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/ci/priv-integration.sh b/ci/priv-integration.sh index 6b5c22aa..431e727d 100755 --- a/ci/priv-integration.sh +++ b/ci/priv-integration.sh @@ -24,6 +24,8 @@ fi if test '!' -d "${sysroot}/ostree/deploy/${stateroot}"; then ostree admin os-init "${stateroot}" --sysroot "${sysroot}" fi +# Should be no images pruned +ostree-ext-cli container image prune-images --sysroot "${sysroot}" # Test the syntax which uses full imgrefs. ostree-ext-cli container image deploy --sysroot "${sysroot}" \ --stateroot "${stateroot}" --imgref "${imgref}" @@ -34,8 +36,11 @@ ostree admin --sysroot="${sysroot}" undeploy 0 ostree-ext-cli container image deploy --transport registry --sysroot "${sysroot}" \ --stateroot "${stateroot}" --image "${image}" --no-signature-verification ostree admin --sysroot="${sysroot}" status -ostree-ext-cli container image remove --repo "${sysroot}/ostree/repo" registry:"${image}" ostree admin --sysroot="${sysroot}" undeploy 0 +# Now we should prune it +ostree-ext-cli container image prune-images --sysroot "${sysroot}" +ostree-ext-cli container image list --repo "${sysroot}/ostree/repo" > out.txt +test $(stat -c '%s' out.txt) = 0 for img in "${image}"; do ostree-ext-cli container image deploy --sysroot "${sysroot}" \ diff --git a/lib/src/cli.rs b/lib/src/cli.rs index acbc973a..62a56cad 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -18,6 +18,7 @@ use crate::commit::container_commit; use crate::container::store::{ImportProgress, LayerProgress, PreparedImport}; use crate::container::{self as ostree_container}; use crate::container::{Config, ImageReference, OstreeImageReference}; +use crate::sysroot::SysrootLock; use ostree_container::store::{ImageImporter, PrepareResult}; /// Parse an [`OstreeImageReference`] from a CLI arguemnt. @@ -273,6 +274,17 @@ pub(crate) enum ContainerImageOpts { repo: Utf8PathBuf, }, + /// Garbage collect unreferenced image layer references. + PruneImages { + /// Path to the system root + #[clap(long)] + sysroot: Utf8PathBuf, + + #[clap(long)] + /// Also prune layers + and_layers: bool, + }, + /// Perform initial deployment for a container image Deploy { /// Path to the system root @@ -825,6 +837,31 @@ where println!("Removed layers: {nlayers}"); Ok(()) } + ContainerImageOpts::PruneImages { + sysroot, + and_layers, + } => { + let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot))); + sysroot.load(gio::Cancellable::NONE)?; + let sysroot = &SysrootLock::new_from_sysroot(sysroot).await?; + let removed = crate::container::deploy::remove_undeployed_images(sysroot)?; + match removed.as_slice() { + [] => { + println!("No unreferenced images."); + return Ok(()); + } + o => { + for imgref in o { + println!("Removed: {imgref}"); + } + } + } + if and_layers { + let nlayers = crate::container::store::gc_image_layers(&sysroot.repo())?; + println!("Removed layers: {nlayers}"); + } + Ok(()) + } ContainerImageOpts::Copy { src_repo, dest_repo, diff --git a/lib/src/container/deploy.rs b/lib/src/container/deploy.rs index 98080c58..431527e3 100644 --- a/lib/src/container/deploy.rs +++ b/lib/src/container/deploy.rs @@ -1,8 +1,12 @@ //! Perform initial setup for a container image based system root +use std::collections::HashSet; + use super::store::LayeredImageState; -use super::OstreeImageReference; +use super::{ImageReference, OstreeImageReference}; use crate::container::store::PrepareResult; +use crate::keyfileext::KeyFileExt; +use crate::sysroot::SysrootLock; use anyhow::Result; use fn_error_context::context; use ostree::glib; @@ -112,3 +116,49 @@ pub async fn deploy( Ok(state) } + +/// Query the container image reference for a deployment +fn deployment_origin_container( + deploy: &ostree::Deployment, +) -> Result> { + let origin = deploy + .origin() + .map(|o| o.optional_string("origin", ORIGIN_CONTAINER)) + .transpose()? + .flatten(); + let r = origin + .map(|v| OstreeImageReference::try_from(v.as_str())) + .transpose()?; + Ok(r) +} + +/// Remove all container images which are not the target of a deployment. +/// This acts equivalently to [`super::store::remove_images()`] - the underlying layers +/// are not pruned. +/// +/// The set of removed images is returned. +pub fn remove_undeployed_images(sysroot: &SysrootLock) -> Result> { + let repo = &sysroot.repo(); + let deployment_origins: Result> = sysroot + .deployments() + .into_iter() + .filter_map(|deploy| { + deployment_origin_container(&deploy) + .map(|v| v.map(|v| v.imgref)) + .transpose() + }) + .collect(); + let deployment_origins = deployment_origins?; + // TODO add an API that returns ImageReference instead + let all_images = super::store::list_images(&sysroot.repo())? + .into_iter() + .filter_map(|img| ImageReference::try_from(img.as_str()).ok()); + let mut removed = Vec::new(); + for image in all_images { + if !deployment_origins.contains(&image) { + super::store::remove_image(repo, &image)?; + removed.push(image); + } + } + Ok(removed) +} diff --git a/lib/src/container/mod.rs b/lib/src/container/mod.rs index 239d7529..cfe0bce9 100644 --- a/lib/src/container/mod.rs +++ b/lib/src/container/mod.rs @@ -48,7 +48,7 @@ pub(crate) const COMPONENT_SEPARATOR: char = ','; type Result = anyhow::Result; /// A backend/transport for OCI/Docker images. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq)] pub enum Transport { /// A remote Docker/OCI registry (`registry:` or `docker://`) Registry, @@ -63,7 +63,7 @@ pub enum Transport { /// Combination of a remote image reference and transport. /// /// For example, -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ImageReference { /// The storage and transport for the image pub transport: Transport, @@ -72,7 +72,7 @@ pub struct ImageReference { } /// Policy for signature verification. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum SignatureSource { /// Fetches will use the named ostree remote for signature verification of the ostree commit. OstreeRemote(String), @@ -87,7 +87,7 @@ pub const LABEL_VERSION: &str = "version"; /// Combination of a signature verification mechanism, and a standard container image reference. /// -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OstreeImageReference { /// The signature verification mechanism. pub sigverify: SignatureSource,