Add new treefile-apply experimental command

We are working towards rebasing FCOS and RHCOS on top of fedora-bootc
and rhel-bootc. As one can imagine, we are quite heavily invested in
treefiles currently, so being able to keep using what we already have as
we transition into becoming a layered build is valuable.

Dockerfiles are great of course as the common language used to build
container images. But it's not great for implementing complex logic.

Add a new `rpm-ostree experimental compose treefile-apply` command
which takes a treefile, but applies its directives _live_ in the running
environment.

So in this model, the business logic in the Dockerfile is kept to a
minimum and all the heavy lifting happens via
`RUN rpm-ostree compose treefile-apply manifest.yaml`.

A very small subset of the treefile spec is actually supported. The goal
is _not_ to reimplement everything, but only what is needed and useful
in a derivation context.

For now, only `packages`, `recommends`, and `postprocess` are supported.
More will be added in the future.

This code actually started as a Python script in openshift/os, currently
used to build the OCP node image:

https://github.com/openshift/os/blob/master/scripts/apply-manifest

So in fact, the node image (which in turn is a layered build on top of
RHCOS) will also be using this code.

ci/test-container.sh

@@ -144,4 +144,18 @@ if ! grep -qe "error: No such file or directory" err.txt; then
     fatal "did not find expected error when skipping CLI wraps."
 fi
 
+# test treefile-apply
+if rpm -q ltrace vim-enhanced; then
+  fatal "ltrace and/or vim-enhanced exist"
+fi
+vim_vr=$(rpm -q vim-minimal --qf '%{version}-%{release}')
+cat > /tmp/treefile.yaml << EOF
+packages:
+  - ltrace
+  # a split base/layered version-locked package
+  - vim-enhanced
+EOF
+rpm-ostree experimental compose treefile-apply /tmp/treefile.yaml
+rpm -q ltrace vim-enhanced-"$vim_vr"
+
 echo ok

rust/src/cli_experimental.rs

@@ -39,6 +39,10 @@ enum ComposeCmd {
         #[clap(flatten)]
         opts: crate::compose::CommitToContainerRootfsOpts,
     },
+    TreefileApply {
+        #[clap(flatten)]
+        opts: crate::treefile::TreefileApplyOpts,
+    },
 }
 
 impl ComposeCmd {
@@ -47,6 +51,7 @@ impl ComposeCmd {
             ComposeCmd::BuildChunkedOCI { opts } => opts.run(),
             ComposeCmd::Rootfs { opts } => opts.run(),
             ComposeCmd::CommitToContainerRootfs { opts } => opts.run(),
+            ComposeCmd::TreefileApply { opts } => opts.run(),
         }
     }
 }

rust/src/composepost.rs

@@ -33,7 +33,7 @@ use std::os::unix::io::AsRawFd;
 use std::os::unix::prelude::IntoRawFd;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
-use std::process::Stdio;
+use std::process::{Command, Stdio};
 
 /// Directories that are moved out and symlinked from their `/var/lib/<entry>`
 /// location to `/usr/lib/<entry>`.
@@ -504,13 +504,18 @@ fn compose_postprocess_default_target(rootfs: &Dir, target: &str) -> Result<()>
     Ok(())
 }
 
+pub(crate) enum PostprocessBwrap {
+    None,
+    Wrap { unified_core: bool },
+}
+
 /// The treefile format has two kinds of postprocessing scripts;
 /// there's a single `postprocess-script` as well as inline (anonymous)
 /// scripts.  This function executes both kinds in bwrap containers.
-fn compose_postprocess_scripts(
+pub(crate) fn compose_postprocess_scripts(
     rootfs_dfd: &Dir,
     treefile: &mut Treefile,
-    unified_core: bool,
+    bwrap: PostprocessBwrap,
 ) -> Result<()> {
     // Execute the anonymous (inline) scripts.
     for (i, script) in treefile
@@ -531,12 +536,19 @@ fn compose_postprocess_scripts(
         )?;
         println!("Executing `postprocess` inline script '{}'", i);
         let child_argv = vec![binpath.to_string()];
-        let _ = bwrap::bubblewrap_run_sync(
-            rootfs_dfd.as_raw_fd(),
-            &child_argv,
-            false,
-            BubblewrapMutability::for_unified_core(unified_core),
-        )?;
+        if let PostprocessBwrap::Wrap { unified_core } = bwrap {
+            let _ = bwrap::bubblewrap_run_sync(
+                rootfs_dfd.as_raw_fd(),
+                &child_argv,
+                false,
+                BubblewrapMutability::for_unified_core(unified_core),
+            )
+            .context("Executing inline postprocessing script")?;
+        } else {
+            Command::new(&binpath)
+                .run()
+                .context("Executing inline postprocessing script")?;
+        }
         rootfs_dfd.remove_file(target_binpath)?;
     }
 
@@ -556,13 +568,19 @@ fn compose_postprocess_scripts(
         println!("Executing postprocessing script");
 
         let child_argv = &vec![binpath.to_string()];
-        let _ = crate::bwrap::bubblewrap_run_sync(
-            rootfs_dfd.as_raw_fd(),
-            child_argv,
-            false,
-            BubblewrapMutability::for_unified_core(unified_core),
-        )
-        .context("Executing postprocessing script")?;
+        if let PostprocessBwrap::Wrap { unified_core } = bwrap {
+            let _ = crate::bwrap::bubblewrap_run_sync(
+                rootfs_dfd.as_raw_fd(),
+                child_argv,
+                false,
+                BubblewrapMutability::for_unified_core(unified_core),
+            )
+            .context("Executing postprocessing script")?;
+        } else {
+            Command::new(&binpath)
+                .run()
+                .context("Executing postprocessing script")?;
+        }
 
         rootfs_dfd.remove_file(target_binpath)?;
         println!("Finished postprocessing script");
@@ -720,7 +738,7 @@ pub fn compose_postprocess(
     compose_postprocess_add_files(rootfs, treefile)?;
     etc_guard.undo()?;
 
-    compose_postprocess_scripts(rootfs, treefile, unified_core)?;
+    compose_postprocess_scripts(rootfs, treefile, PostprocessBwrap::Wrap { unified_core })?;
 
     Ok(())
 }

rust/src/treefile.rs

@@ -20,13 +20,14 @@
  */
 
 use crate::cmdutils::CommandRunExt;
-use crate::cxxrsutil::*;
+use crate::{compose_postprocess_scripts, cxxrsutil::*};
 use anyhow::{anyhow, bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
 use cap_std::fs::MetadataExt as _;
 use cap_std_ext::cap_std::fs::Dir;
 use cap_std_ext::cmdext::CapStdExtCommandExt;
 use cap_std_ext::prelude::CapStdExtDirExt;
+use clap::Parser;
 use fn_error_context::context;
 use nix::unistd::{Gid, Uid};
 use once_cell::sync::Lazy;
@@ -4673,3 +4674,67 @@ pub(crate) fn treefile_delete_client_etc() -> CxxResult<u32> {
     }
     Ok(n)
 }
+
+/// Apply a treefile to the running environment (usually during an image build)
+#[derive(Debug, Parser)]
+pub(crate) struct TreefileApplyOpts {
+    /// Path to the treefile.
+    #[clap(value_parser)]
+    treefile: Utf8PathBuf,
+}
+
+impl TreefileApplyOpts {
+    pub(crate) fn run(self) -> Result<()> {
+        let mut tf = Treefile::new_boxed(&self.treefile, Some(&utils::get_rpm_basearch()))
+            .context("parsing treefile")?;
+
+        // we only handle a subset of fields here
+        if let Some(packages) = &tf.parsed.packages {
+            let mut install_args: Vec<_> = packages.iter().map(|s| s.as_str()).collect();
+
+            if tf.parsed.documentation.is_some() {
+                bail!("cannot apply documentation directive when deriving");
+            }
+
+            // map `recommends` to dnf's `install_weak_deps`
+            match tf.parsed.recommends {
+                Some(true) => {
+                    install_args.push("--setopt=install_weak_deps=true");
+                }
+                Some(false) => {
+                    install_args.push("--setopt=install_weak_deps=false");
+                }
+                None => {} // implicitly leave environment default if unset
+            }
+
+            // lock all base packages during installation
+            // https://gitlab.com/fedora/bootc/tracker/-/issues/59
+            run_dnf("versionlock", &["add", "*", "--disablerepo", "*"])
+                .context("locking base packages with dnf")?;
+            run_dnf("install", &install_args).context("installing packages with dnf")?;
+            run_dnf("versionlock", &["clear"]).context("clearing base packages lock with dnf")?;
+        };
+
+        // handle postprocess scripts
+        let rootfs_dfd =
+            Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("opening /")?;
+        compose_postprocess_scripts(&rootfs_dfd, &mut tf, crate::PostprocessBwrap::None)
+            .context("running postprocess scripts")?;
+
+        Ok(())
+    }
+}
+
+fn run_dnf(command: &str, args: &[&str]) -> Result<()> {
+    let mut cmd = Command::new("dnf");
+    cmd.arg(command).args(args);
+    cmd.arg("--noplugins");
+    cmd.arg("-y");
+
+    let status = cmd.status().context("collecting dnf status")?;
+    if !status.success() {
+        bail!("Failed to run dnf {command}: {status:?}");
+    }
+
+    Ok(())
+}