Add bootc job

.gitignore

@@ -14,11 +14,16 @@ __pycache__
 
 /docs/osbuild-composer.7
 .cache
-container_composer_golangci_built.info
 processed-templates
 
 coverage.txt
 coverage.html
-
 coverage_splunk_logger.txt
 coverage_splunk_logger.html
+
+go.local.mod
+go.local.sum
+container_worker_built.info
+container_composer_built.info
+container_composer_golangci_built.info
+

Makefile

@@ -169,6 +169,10 @@ clean:
 	rm -rf $(CURDIR)/rpmbuild
 	rm -rf container_composer_golangci_built.info
 	rm -rf $(BUILDDIR)/$(PROCESSED_TEMPLATE_DIR)
+	rm -rf $(BUILDDIR)/build/
+	rm -f $(BUILDDIR)/go.local.*
+	rm -f $(BUILDDIR)/container_worker_built.info
+	rm -f $(BUILDDIR)/container_composer_built.info
 
 .PHONY: push-check
 push-check: lint build unit-tests srpm man
@@ -326,3 +330,97 @@ $(PROCESSED_TEMPLATE_DIR)/%.yml: $(PROCESSED_TEMPLATE_DIR) $(OPENSHIFT_TEMPLATES
 .PHONY: process-templates
 process-templates: $(addprefix $(PROCESSED_TEMPLATE_DIR)/, $(OPENSHIFT_TEMPLATES))
 
+# either "docker" or "sudo podman"
+# podman needs to build as root as it also needs to run as root afterwards
+CONTAINER_EXECUTABLE ?= sudo podman
+
+DOCKER_IMAGE_WORKER := osbuild-worker_devel
+DOCKERFILE_WORKER := distribution/Dockerfile-worker_srcinstall
+
+DOCKER_IMAGE_COMPOSER := osbuild-composer_devel
+DOCKERFILE_COMPOSER := distribution/Dockerfile-composer_srcinstall
+
+GOPROXY ?= https://proxy.golang.org,direct
+
+# source where the other repos are locally
+# has to end with a trailing slash
+SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../
+
+# names of folder that have to be git-cloned additionally to be able
+# to build all code
+SRC_DEPS_EXTERNAL_NAMES := images pulp-client
+SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SRC_DEPS_EXTERNAL_NAMES))
+
+$(SRC_DEPS_EXTERNAL_DIRS):
+	@for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done
+
+
+SRC_DEPS_DIRS := internal cmd pkg repositories
+
+# All files to check for rebuild!
+SRC_DEPS := $(shell find $(SRC_DEPS_DIRS) -name *.go -or -name *.json)
+SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go)
+
+# dependencies to rebuild worker
+WORKER_SRC_DEPS := $(SRC_DEPS)
+# dependencies to rebuild composer
+COMPOSER_SRC_DEPS := $(SRC_DEPS)
+
+GOMODARGS ?= -modfile=go.local.mod
+# gcflags "-N -l" for golang to allow debugging
+GCFLAGS ?= -gcflags=all=-N -gcflags=all=-l
+
+CONTAINER_DEPS_COMPOSER := ./containers/osbuild-composer/entrypoint.py
+CONTAINER_DEPS_WORKER := ./distribution/osbuild-worker-entrypoint.sh
+
+USE_BTRFS ?= yes
+
+
+# source where the other repos are locally
+# has to end with a trailing slash
+SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../
+
+COMMON_SRC_DEPS_NAMES := osbuild
+COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES))
+
+OSBUILD_CONTAINER_INDICATOR := $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild/container_built.info
+
+CONTAINER_EXECUTABLE ?= docker
+MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)"
+
+$(COMMON_SRC_DEPS_ORIGIN):
+	@for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done
+
+# we'll trigger the sub-make for osbuild with "osbuild-container"
+# and use OSBUILD_CONTAINER_INDICATOR to check if we need to rebuild our containers here
+.PHONY: osbuild-container
+$(OSBUILD_CONTAINER_INDICATOR) osbuild-container:
+	$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild container
+
+go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(WORKER_SRC_DEPS) $(COMPOSER_SRC_DEPS) Makefile
+	cp go.mod go.local.mod
+	cp go.sum go.local.sum
+
+	go mod edit $(GOMODARGS) -replace github.com/osbuild/images=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)images
+	go mod edit $(GOMODARGS) -replace github.com/osbuild/pulp-client=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)pulp-client
+	go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=./pkg/splunk_logger
+	env GOPROXY=$(GOPROXY) go mod tidy $(GOMODARGS)
+	env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS)
+
+container_worker_built.info: go.local.mod $(WORKER_SRC_DEPS) $(DOCKERFILE_WORKER) $(CONTAINER_DEPS_WORKER) $(OSBUILD_CONTAINER_INDICATOR)
+	$(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_WORKER) -f $(DOCKERFILE_WORKER) --build-arg GOMODARGS="$(GOMODARGS)"  --build-arg GCFLAGS="$(GCFLAGS)" --build-arg USE_BTRFS=$(USE_BTRFS) .
+	echo "Worker last built on" > $@
+	date >> $@
+
+container_composer_built.info: go.local.mod $(COMPOSER_SRC_DEPS) $(DOCKERFILE_COMPOSER) $(CONTAINER_DEPS_COMPOSER) $(OSBUILD_CONTAINER_INDICATOR)
+	$(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_COMPOSER) -f $(DOCKERFILE_COMPOSER) --build-arg GOMODARGS="$(GOMODARGS)" --build-arg GCFLAGS="$(GCFLAGS)" .
+	echo "Composer last built on" > $@
+	date >> $@
+
+# build a container with a worker from full source
+.PHONY: container_worker
+container_worker: osbuild-container container_worker_built.info
+
+# build a container with the composer from full source
+.PHONY: container_composer
+container_composer: osbuild-container container_composer_built.info

cmd/osbuild-worker/jobimpl-bootc.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/osbuild/osbuild-composer/internal/worker"
+	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
+)
+
+type BootcImageBuilderJobImpl struct{}
+
+type ErrorInvalidImageType string
+
+func (e ErrorInvalidImageType) Error() string { return string(e) }
+
+type ErrorCommandExecution string
+
+func (e ErrorCommandExecution) Error() string { return string(e) }
+
+type ErrorInvalidArguments string
+
+func (e ErrorInvalidArguments) Error() string { return string(e) }
+
+type ErrorUnknown string
+
+func (e ErrorUnknown) Error() string { return string(e) }
+
+func setResponseError(err error, result *worker.BootcImageBuilderJobResult) {
+	switch e := err.(type) {
+	case ErrorInvalidImageType:
+		result.JobError = clienterrors.New(
+			clienterrors.ErrorInvalidImageType,
+			"Unsupported image type",
+			e.Error(),
+		)
+	case ErrorCommandExecution:
+		result.JobError = clienterrors.New(
+			clienterrors.ErrorBootcExecution,
+			"Error running bootc-image-builder command",
+			e.Error(),
+		)
+	default:
+		result.JobError = clienterrors.New(
+			clienterrors.ErrorInvalidArguments,
+			"Invalid bootc parameters or parameter combination",
+			err.Error(),
+		)
+	}
+}
+
+func (impl *BootcImageBuilderJobImpl) Run(job worker.Job) error {
+	logrus.Info("Running bootc-image-builder job")
+	logWithId := logrus.WithField("jobId", job.Id())
+	// Parse the job arguments
+	var args worker.BootcImageBuilderJob
+	err := job.Args(&args)
+	if err != nil {
+		return err
+	}
+
+	// Initialize the result
+	jobResult := worker.BootcImageBuilderJobResult{}
+
+	// Determine the output image type
+	imageType := ""
+	switch args.ImageType {
+	case "ami":
+		imageType = "ami"
+	case "qcow2":
+		imageType = "qcow2"
+	case "aws-bootc":
+		imageType = "ami"
+	case "guest-image-bootc":
+		imageType = "qcow2"
+	default:
+		err := ErrorInvalidImageType(fmt.Sprintf("Unsupported image type: %v", args.ImageType))
+		setResponseError(err, &jobResult)
+		return job.Update(&jobResult)
+	}
+
+	// Construct the podman command with bootc-image-builder
+	cmd := exec.Command(
+		"sudo", "podman", "run",
+		"--rm",
+		"--privileged",
+		"--pull=missing",
+		"--storage-driver=vfs",
+		"--cgroup-manager=cgroupfs",
+		"--runtime=runc",
+		"--security-opt", "label=type:unconfined_t",
+		"-v", "/var/cache/osbuild-composer/output:/output",
+		"quay.io/centos-bootc/bootc-image-builder:latest",
+		args.ImageRef,
+	)
+
+	// Append optional flags
+	if imageType != "" {
+		cmd.Args = append(cmd.Args, "--type", imageType)
+	}
+	if args.Arch != "" {
+		cmd.Args = append(cmd.Args, "--target-arch", args.Arch)
+	}
+	if args.Chown != "" {
+		cmd.Args = append(cmd.Args, "--chown", args.Chown)
+	}
+	if !args.TLSVerify {
+		cmd.Args = append(cmd.Args, "--tls-verify=false")
+	}
+
+	// Log the command being run
+	logWithId.Infof("Running command: %s", cmd.String())
+
+	// Execute the command
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		logWithId.Errorf("Error running bootc-image-builder: %v, Output: %s", err, string(output))
+		setResponseError(ErrorCommandExecution(err.Error()), &jobResult)
+		return job.Update(&jobResult)
+	}
+
+	// Log the output of the command
+	logWithId.Infof("bootc-image-builder output: %s", string(output))
+
+	// If the command succeeded, update the job result
+	jobResult.JobResult = worker.JobResult{} // No error
+	err = job.Update(&jobResult)
+	if err != nil {
+		setResponseError(ErrorUnknown(fmt.Sprintf("Error reporting job result: %v", err)), &jobResult)
+		return job.Update(&jobResult)
+	}
+
+	return nil
+}

cmd/osbuild-worker/main.go

@@ -7,13 +7,14 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger"
 	"net/url"
 	"os"
 	"path"
 	"strings"
 	"time"
 
+	slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger"
+
 	"github.com/BurntSushi/toml"
 	"github.com/sirupsen/logrus"
 
@@ -516,6 +517,7 @@ func main() {
 		worker.JobTypeAWSEC2Share: &AWSEC2ShareJobImpl{
 			AWSCreds: awsCredentials,
 		},
+		worker.JobTypeBootableContainer: &BootcImageBuilderJobImpl{},
 	}
 
 	acceptedJobTypes := []string{}