build: add cargo-based build system

Signed-off-by: Ben Cressey <[email protected]>
Signed-off-by: Zac Mrowicki <[email protected]>

tools/buildsys/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "buildsys"
+version = "0.1.0"
+authors = ["Ben Cressey <[email protected]>"]
+edition = "2018"
+publish = false
+
+[dependencies]
+duct = "0.12.0"
+hex = "0.3"
+rand = { version = "0.7", default-features = false, features = ["std"] }
+reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+sha2 = "0.8"
+snafu = "0.5"
+toml = "0.5"
+users = { version = "0.9", default-features = false }
+walkdir = "2"

tools/buildsys/src/builder.rs

@@ -0,0 +1,144 @@
+/*!
+This module handles the calls to the BuildKit server needed to execute package
+and image builds. The actual build steps and the expected parameters are defined
+in the repository's top-level Dockerfile.
+
+*/
+pub(crate) mod error;
+use error::Result;
+
+use duct::cmd;
+use rand::Rng;
+use snafu::ResultExt;
+use std::env;
+use std::process::Output;
+use users::get_effective_uid;
+
+pub(crate) struct PackageBuilder;
+
+impl PackageBuilder {
+    /// Call `buildctl` to produce RPMs for the specified package.
+    pub(crate) fn build(package: &str) -> Result<(Self)> {
+        let arch = getenv("BUILDSYS_ARCH")?;
+        let opts = format!(
+            "--opt target=rpm \
+             --opt build-arg:PACKAGE={package} \
+             --opt build-arg:ARCH={arch}",
+            package = package,
+            arch = arch,
+        );
+
+        let result = buildctl(&opts)?;
+        if !result.status.success() {
+            let output = String::from_utf8_lossy(&result.stdout);
+            return error::PackageBuild { package, output }.fail();
+        }
+
+        Ok(Self)
+    }
+}
+
+pub(crate) struct ImageBuilder;
+
+impl ImageBuilder {
+    /// Call `buildctl` to create an image with the specified packages installed.
+    pub(crate) fn build(packages: &[String]) -> Result<(Self)> {
+        // We want PACKAGES to be a value that contains spaces, since that's
+        // easier to work with in the shell than other forms of structured data.
+        let packages = packages.join("|");
+
+        let arch = getenv("BUILDSYS_ARCH")?;
+        let opts = format!(
+            "--opt target=image \
+             --opt build-arg:PACKAGES={packages} \
+             --opt build-arg:ARCH={arch}",
+            packages = packages,
+            arch = arch,
+        );
+
+        // Always rebuild images since they are located in a different workspace,
+        // and don't directly track changes in the underlying packages.
+        getenv("BUILDSYS_TIMESTAMP")?;
+
+        let result = buildctl(&opts)?;
+        if !result.status.success() {
+            let output = String::from_utf8_lossy(&result.stdout);
+            return error::ImageBuild { packages, output }.fail();
+        }
+
+        Ok(Self)
+    }
+}
+
+/// Invoke `buildctl` by way of `docker` with the arguments for a specific
+/// package or image build.
+fn buildctl(opts: &str) -> Result<Output> {
+    let docker_args = docker_args()?;
+    let buildctl_args = buildctl_args()?;
+
+    // Avoid using a cached layer from a previous build.
+    let nocache = format!(
+        "--opt build-arg:NOCACHE={}",
+        rand::thread_rng().gen::<u32>(),
+    );
+
+    // Build the giant chain of args. Treat "|" as a placeholder that indicates
+    // where the argument should contain spaces after we split on whitespace.
+    let args = docker_args
+        .split_whitespace()
+        .chain(buildctl_args.split_whitespace())
+        .chain(opts.split_whitespace())
+        .chain(nocache.split_whitespace())
+        .map(|s| s.replace("|", " "));
+
+    // Run the giant docker invocation
+    cmd("docker", args)
+        .stderr_to_stdout()
+        .run()
+        .context(error::CommandExecution)
+}
+
+/// Prepare the arguments for docker
+fn docker_args() -> Result<String> {
+    // Gather the user context.
+    let uid = get_effective_uid();
+
+    // Gather the environment context.
+    let root_dir = getenv("BUILDSYS_ROOT_DIR")?;
+    let buildkit_client = getenv("BUILDSYS_BUILDKIT_CLIENT")?;
+
+    let docker_args = format!(
+        "run --init --rm --network host --user {uid}:{uid} \
+         --volume {root_dir}:{root_dir} --workdir {root_dir} \
+         --entrypoint /usr/bin/buildctl {buildkit_client}",
+        uid = uid,
+        root_dir = root_dir,
+        buildkit_client = buildkit_client
+    );
+
+    Ok(docker_args)
+}
+
+fn buildctl_args() -> Result<String> {
+    // Gather the environment context.
+    let output_dir = getenv("BUILDSYS_OUTPUT_DIR")?;
+    let buildkit_server = getenv("BUILDSYS_BUILDKIT_SERVER")?;
+
+    let buildctl_args = format!(
+        "--addr {buildkit_server} build --progress=plain \
+         --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
+         --output type=local,dest={output_dir}",
+        buildkit_server = buildkit_server,
+        output_dir = output_dir
+    );
+
+    Ok(buildctl_args)
+}
+
+/// Retrieve a BUILDSYS_* variable that we expect to be set in the environment,
+/// and ensure that we track it for changes, since it will directly affect the
+/// output.
+fn getenv(var: &str) -> Result<String> {
+    println!("cargo:rerun-if-env-changed={}", var);
+    env::var(var).context(error::Environment { var })
+}

tools/buildsys/src/builder/error.rs

@@ -0,0 +1,22 @@
+use snafu::Snafu;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility = "pub(super)")]
+pub enum Error {
+    #[snafu(display("Failed to execute command: {}", source))]
+    CommandExecution { source: std::io::Error },
+
+    #[snafu(display("Failed to build package '{}':\n{}", package, output,))]
+    PackageBuild { package: String, output: String },
+
+    #[snafu(display("Failed to build image with '{}':\n{}", packages, output,))]
+    ImageBuild { packages: String, output: String },
+
+    #[snafu(display("Missing environment variable '{}'", var))]
+    Environment {
+        var: String,
+        source: std::env::VarError,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;

tools/buildsys/src/cache.rs

@@ -0,0 +1,112 @@
+/*!
+Many of the inputs to package builds are not source files tracked within the git
+repository, but large binary artifacts such as tar archives that are independently
+distributed by an upstream project.
+
+This module provides the ability to retrieve and validate these external files,
+given the (name, url, hash) data that uniquely identifies each file.
+
+It implements a two-tier approach to retrieval: files are first pulled from the
+"lookaside" cache and only fetched from the upstream site if that access fails.
+
+*/
+pub(crate) mod error;
+use error::Result;
+
+use super::manifest;
+use sha2::{Digest, Sha512};
+use snafu::{ensure, ResultExt};
+use std::env;
+use std::fs::{self, File};
+use std::io::{self, BufWriter};
+use std::path::{Path, PathBuf};
+
+static LOOKASIDE_CACHE: &str = "https://thar-upstream-lookaside-cache.s3.us-west-2.amazonaws.com";
+
+pub(crate) struct LookasideCache;
+
+impl LookasideCache {
+    /// Fetch files stored out-of-tree and ensure they match the stored hash.
+    pub(crate) fn fetch(files: &[manifest::ExternalFile]) -> Result<Self> {
+        for f in files {
+            let path = &f.path;
+            ensure!(
+                path.components().count() == 1,
+                error::ExternalFileName { path }
+            );
+
+            let hash = &f.hash;
+            if path.is_file() {
+                match Self::verify_file(path, hash) {
+                    Ok(_) => continue,
+                    Err(e) => {
+                        eprintln!("{}", e);
+                        fs::remove_file(path).context(error::ExternalFileDelete { path })?;
+                    }
+                }
+            }
+
+            let name = path.display();
+            let tmp = PathBuf::from(format!(".{}", name));
+
+            // first check the lookaside cache
+            let url = format!("{}/{}/{}/{}", LOOKASIDE_CACHE.to_string(), name, hash, name);
+            match Self::fetch_file(&url, &tmp, hash) {
+                Ok(_) => {
+                    fs::rename(&tmp, path).context(error::ExternalFileRename { path: &tmp })?;
+                    continue;
+                }
+                Err(e) => {
+                    eprintln!("{}", e);
+                }
+            }
+
+            // next check with upstream, if permitted
+            if env::var_os("BUILDSYS_ALLOW_UPSTREAM_SOURCE_URL").is_some() {
+                Self::fetch_file(&f.url, &tmp, hash)?;
+                fs::rename(&tmp, path).context(error::ExternalFileRename { path: &tmp })?;
+            }
+        }
+
+        Ok(Self)
+    }
+
+    /// Retrieves a file from the specified URL and write it to the given path,
+    /// then verifies the contents against the hash provided.
+    fn fetch_file<P: AsRef<Path>>(url: &str, path: P, hash: &str) -> Result<()> {
+        let path = path.as_ref();
+        let mut resp = reqwest::get(url).context(error::ExternalFileUrlRequest { url })?;
+        let status = resp.status();
+        ensure!(
+            status.is_success(),
+            error::ExternalFileUrlFetch { url, status }
+        );
+
+        let f = File::create(path).context(error::ExternalFileOpen { path })?;
+        let mut f = BufWriter::new(f);
+        resp.copy_to(&mut f)
+            .context(error::ExternalFileSave { path })?;
+        drop(f);
+
+        match Self::verify_file(path, hash) {
+            Ok(_) => Ok(()),
+            Err(e) => {
+                fs::remove_file(path).context(error::ExternalFileDelete { path })?;
+                Err(e)
+            }
+        }
+    }
+
+    /// Reads a file from disk and compares it to the expected SHA-512 hash.
+    fn verify_file<P: AsRef<Path>>(path: P, hash: &str) -> Result<()> {
+        let path = path.as_ref();
+        let mut f = File::open(path).context(error::ExternalFileOpen { path })?;
+        let mut d = Sha512::new();
+
+        io::copy(&mut f, &mut d).context(error::ExternalFileLoad { path })?;
+        let digest = hex::encode(d.result());
+
+        ensure!(digest == hash, error::ExternalFileVerify { path, hash });
+        Ok(())
+    }
+}

tools/buildsys/src/cache/error.rs

@@ -0,0 +1,42 @@
+use snafu::Snafu;
+use std::io;
+use std::path::PathBuf;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility = "pub")]
+pub enum Error {
+    #[snafu(display("Bad file name '{}'", path.display()))]
+    ExternalFileName { path: PathBuf },
+
+    #[snafu(display("Failed to request '{}': {}", url, source))]
+    ExternalFileUrlRequest { url: String, source: reqwest::Error },
+
+    #[snafu(display("Failed to fetch '{}': {}", url, status))]
+    ExternalFileUrlFetch {
+        url: String,
+        status: reqwest::StatusCode,
+    },
+
+    #[snafu(display("Failed to open file '{}': {}", path.display(), source))]
+    ExternalFileOpen { path: PathBuf, source: io::Error },
+
+    #[snafu(display("Failed to write file '{}': {}", path.display(), source))]
+    ExternalFileSave {
+        path: PathBuf,
+        source: reqwest::Error,
+    },
+
+    #[snafu(display("Failed to load file '{}': {}", path.display(), source))]
+    ExternalFileLoad { path: PathBuf, source: io::Error },
+
+    #[snafu(display("Failed to verify file '{}' with hash '{}'", path.display(), hash))]
+    ExternalFileVerify { path: PathBuf, hash: String },
+
+    #[snafu(display("Failed to rename file '{}': {}", path.display(), source))]
+    ExternalFileRename { path: PathBuf, source: io::Error },
+
+    #[snafu(display("Failed to delete file '{}': {}", path.display(), source))]
+    ExternalFileDelete { path: PathBuf, source: io::Error },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;