diff --git a/Containerfile b/Containerfile index 9490b1f4..bea48230 100644 --- a/Containerfile +++ b/Containerfile @@ -23,4 +23,3 @@ VOLUME /output WORKDIR /output VOLUME /store VOLUME /rpmmd - diff --git a/README.md b/README.md index 35875ef8..39d6e545 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,33 @@ sudo podman run \ quay.io/centos-bootc/fedora-bootc:eln ``` +### Using local containers + +To use containers from local container's storage rather than a registry, we need to ensure two things: +- the container exists in local storage +- mount the local container storage + +Since the container is run in `rootful` only root container storage paths are allowed. + +```bash +sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v $(pwd)/config.json:/config.json \ + -v $(pwd)/output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + --config /config.json \ + --local \ + localhost/bootc:eln +``` + +When using the --local flag, we need to mount the storage path as a volume. With this enabled, it is assumed that the target container is in the container storage. + ### Running the resulting QCOW2 file on Linux (x86_64) A virtual machine can be launched using `qemu-system-x86_64` or with `virt-install` as shown below. diff --git a/bib/cmd/bootc-image-builder/image.go b/bib/cmd/bootc-image-builder/image.go index de9d9b44..9e5c22f9 100644 --- a/bib/cmd/bootc-image-builder/image.go +++ b/bib/cmd/bootc-image-builder/image.go @@ -42,6 +42,9 @@ type ManifestConfig struct { // TLSVerify specifies whether HTTPS and a valid TLS certificate are required TLSVerify bool + + // Use a local container from the host rather than a repository + Local bool } func Manifest(c *ManifestConfig) (*manifest.Manifest, error) { @@ -65,6 +68,7 @@ func manifestForDiskImage(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest Source: c.Imgref, Name: c.Imgref, TLSVerify: &c.TLSVerify, + Local: c.Local, } var customizations *blueprint.Customizations @@ -142,6 +146,7 @@ func manifestForDiskImage(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest Source: c.Imgref, Name: c.Imgref, TLSVerify: &c.TLSVerify, + Local: c.Local, }, } _, err = img.InstantiateManifestFromContainers(&mf, containerSources, runner, rng) @@ -158,6 +163,7 @@ func manifestForISO(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest, erro Source: c.Imgref, Name: c.Imgref, TLSVerify: &c.TLSVerify, + Local: c.Local, } // The ref is not needed and will be removed from the ctor later diff --git a/bib/cmd/bootc-image-builder/main.go b/bib/cmd/bootc-image-builder/main.go index 96f2166c..9933eac9 100644 --- a/bib/cmd/bootc-image-builder/main.go +++ b/bib/cmd/bootc-image-builder/main.go @@ -187,6 +187,7 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { rpmCacheRoot, _ := cmd.Flags().GetString("rpmmd") targetArch, _ := cmd.Flags().GetString("target-arch") tlsVerify, _ := cmd.Flags().GetBool("tls-verify") + localStorage, _ := cmd.Flags().GetBool("local") // translate anaconda-iso to iso to avoid multiple image type checks for idx := range imgTypes { @@ -240,6 +241,7 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { Imgref: imgref, Repos: repos, TLSVerify: tlsVerify, + Local: localStorage, } return makeManifest(manifestConfig, rpmCacheRoot) } @@ -422,6 +424,7 @@ func run() error { manifestCmd.Flags().String("rpmmd", "/rpmmd", "rpm metadata cache directory") manifestCmd.Flags().String("target-arch", "", "build for the given target architecture (experimental)") manifestCmd.Flags().StringArray("type", []string{"qcow2"}, "image types to build [qcow2, ami, iso, raw]") + manifestCmd.Flags().Bool("local", false, "use a local container rather than a container from a registry") logrus.SetLevel(logrus.ErrorLevel) buildCmd.Flags().AddFlagSet(manifestCmd.Flags()) diff --git a/build.sh b/build.sh index 400ff086..8d5ed9aa 100755 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ set -euo pipefail # Keep this in sync with e.g. https://github.com/containers/podman/blob/2981262215f563461d449b9841741339f4d9a894/Makefile#L51 # It turns off the esoteric containers-storage backends that add dependencies # on things like btrfs that we don't need. -CONTAINERS_STORAGE_THIN_TAGS="containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay" +CONTAINERS_STORAGE_THIN_TAGS="containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper" cd bib set -x diff --git a/plans/all.fmf b/plans/all.fmf index 3737a5e5..718dea0d 100644 --- a/plans/all.fmf +++ b/plans/all.fmf @@ -19,6 +19,7 @@ prepare: - python3-flake8 - python3-paramiko - python3-pip + - skopeo - qemu-kvm - qemu-system-aarch64 - qemu-user-static diff --git a/test/test_build.py b/test/test_build.py index 6ee56fba..8d77784c 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -2,9 +2,11 @@ import os import pathlib import platform +import random import re import shutil import subprocess +import string import tempfile import uuid from contextlib import contextmanager @@ -42,6 +44,24 @@ class ImageBuildResult(NamedTuple): metadata: dict = {} +def parse_request_params(request): + # image_type is passed via special pytest parameter fixture + testcase_ref = request.param + if testcase_ref.count(",") == 3: + container_ref, images, target_arch, local = testcase_ref.split(",") + local = local is not None + elif testcase_ref.count(",") == 2: + container_ref, images, target_arch = testcase_ref.split(",") + local = False + elif testcase_ref.count(",") == 1: + container_ref, images = testcase_ref.split(",") + target_arch = None + local = False + else: + raise ValueError(f"cannot parse {testcase_ref.count}") + return container_ref, images, target_arch, local + + @pytest.fixture(scope='session') def shared_tmpdir(tmpdir_factory): tmp_path = pathlib.Path(tmpdir_factory.mktemp("shared")) @@ -53,9 +73,29 @@ def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload """ Build an image inside the passed build_container and return an ImageBuildResult with the resulting image path and user/password + In the case an image is being built from a local container, the + function will build the required local container for the test. """ - with build_images(shared_tmpdir, build_container, request, force_aws_upload) as build_results: - yield build_results[0] + container_ref, images, target_arch, local = parse_request_params(request) + + if not local: + with build_images(shared_tmpdir, build_container, request, force_aws_upload) as build_results: + yield build_results[0] + else: + cont_tag = "localhost/cont-base-" + "".join(random.choices(string.digits, k=12)) + + # we are not cross-building local images (for now) + request.param = ",".join([cont_tag, images, "", "true"]) + + # copy the container into containers-storage + subprocess.check_call([ + "skopeo", "copy", + f"docker://{container_ref}", + f"containers-storage:[overlay@/var/lib/containers/storage+/run/containers/storage]{cont_tag}" + ]) + + with build_images(shared_tmpdir, build_container, request, force_aws_upload) as build_results: + yield build_results[0] @pytest.fixture(name="images", scope="session") @@ -76,17 +116,9 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): Will return cached results of previous build requests. - :request.parm: has the form "container_url,img_type1+img_type2,arch" + :request.param: has the form "container_url,img_type1+img_type2,arch,local" """ - # image_type is passed via special pytest parameter fixture - testcase_ref = request.param - if testcase_ref.count(",") == 2: - container_ref, images, target_arch = testcase_ref.split(",") - elif testcase_ref.count(",") == 1: - container_ref, images = testcase_ref.split(",") - target_arch = None - else: - raise ValueError(f"cannot parse {testcase_ref.count}") + container_ref, images, target_arch, local = parse_request_params(request) # images might be multiple --type args # split and check each one @@ -205,6 +237,13 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): "--security-opt", "label=type:unconfined_t", "-v", f"{output_path}:/output", "-v", "/store", # share the cache between builds + ] + + # we need to mount the host's container store + if local: + cmd.extend(["-v", "/var/lib/containers/storage:/var/lib/containers/storage"]) + + cmd.extend([ *creds_args, build_container, container_ref, @@ -212,7 +251,9 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): *types_arg, *upload_args, *target_arch_args, - ] + "--local" if local else "--local=false", + ]) + # print the build command for easier tracing print(" ".join(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) diff --git a/test/testcases.py b/test/testcases.py index 86869805..923c716d 100644 --- a/test/testcases.py +++ b/test/testcases.py @@ -45,6 +45,8 @@ def gen_testcases(what): CONTAINERS_TO_TEST["fedora"] + "," + DIRECT_BOOT_IMAGE_TYPES[2], CONTAINERS_TO_TEST["centos"] + "," + DIRECT_BOOT_IMAGE_TYPES[2], CONTAINERS_TO_TEST["fedora"] + "," + DIRECT_BOOT_IMAGE_TYPES[0], + CONTAINERS_TO_TEST["centos"] + "," + DIRECT_BOOT_IMAGE_TYPES[0] + ",,true", + CONTAINERS_TO_TEST["fedora"] + "," + DIRECT_BOOT_IMAGE_TYPES[2] + ",,true", ] # do a cross arch test too if platform.machine() == "x86_64":