tests: kubernetes worker

.github/workflows/build.yml

@@ -60,6 +60,7 @@ jobs:
         worker:
           - docker-container
           - remote
+          - kubernetes
         pkg:
           - ./tests
         mode:

Dockerfile

@@ -13,6 +13,7 @@ ARG GOTESTSUM_VERSION=v1.12.0
 ARG REGISTRY_VERSION=3.0.0
 ARG BUILDKIT_VERSION=v0.23.2
 ARG UNDOCK_VERSION=0.9.0
+ARG K3S_VERSION=v1.32.1-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -25,6 +26,7 @@ FROM dockereng/cli-bin:$DOCKER_VERSION_ALT_26 AS docker-cli-alt26
 FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
+FROM rancher/k3s:${K3S_VERSION} AS k3s
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -127,6 +129,17 @@ RUN apk add --no-cache \
       shadow-uidmap \
       xfsprogs \
       xz
+# k3s deps
+RUN apk add --no-cache \
+      busybox-binsh \
+      cni-plugins \
+      cni-plugin-flannel \
+      conntrack-tools \
+      coreutils \
+      dbus \
+      findutils \
+      ipset
+ENV PATH="/usr/libexec/cni:${PATH}"
 COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
@@ -135,11 +148,30 @@ COPY --link --from=docker-engine-alt27 / /opt/docker-alt-27/
 COPY --link --from=docker-engine-alt26 / /opt/docker-alt-26/
 COPY --link --from=docker-cli-alt27 / /opt/docker-alt-27/
 COPY --link --from=docker-cli-alt26 / /opt/docker-alt-26/
+COPY --link --from=k3s /bin/k3s /usr/bin/
+COPY --link --from=k3s /bin/kubectl /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
+COPY --chmod=755 <<-"EOF" /entrypoint.sh
+#!/bin/sh
+set -e
+# cgroup v2: enable nesting
+# https://github.com/moby/moby/blob/v25.0.0/hack/dind#L59-L69
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+  # move the processes from the root group to the /init group,
+  # otherwise writing subtree_control fails with EBUSY.
+  # An error during moving non-existent process (i.e., "cat") is ignored.
+  mkdir -p /sys/fs/cgroup/init
+  xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  # enable controllers
+  sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers > /sys/fs/cgroup/cgroup.subtree_control
+fi
+exec "$@"
+EOF
 ENV TEST_DOCKER_EXTRA="[email protected]=/opt/docker-alt-27,[email protected]=/opt/docker-alt-26"
+ENTRYPOINT ["/entrypoint.sh"]
 
 FROM integration-test-base AS integration-test
 COPY . .

go.mod

@@ -6,9 +6,10 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/compose-spec/compose-go/v2 v2.7.1
 	github.com/containerd/console v1.0.5
-	github.com/containerd/containerd/v2 v2.1.1
+	github.com/containerd/containerd/v2 v2.1.3
 	github.com/containerd/continuity v0.4.5
 	github.com/containerd/errdefs v1.0.0
 	github.com/containerd/log v0.1.0
@@ -87,7 +88,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
 	github.com/aws/smithy-go v1.20.3 // indirect
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/containerd/containerd/api v1.9.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/ttrpc v1.2.7 // indirect
@@ -182,3 +182,5 @@ exclude (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 )
+
+replace github.com/moby/buildkit => github.com/crazy-max/buildkit v0.7.1-0.20250628122957-0cbe82310f5c // testutil-expose-mirror

go.sum

@@ -70,8 +70,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0=
 github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI=
-github.com/containerd/containerd/v2 v2.1.1 h1:znnkm7Ajz8lg8BcIPMhc/9yjBRN3B+OkNKqKisKfwwM=
-github.com/containerd/containerd/v2 v2.1.1/go.mod h1:zIfkQj4RIodclYQkX7GSSswSwgP8d/XxDOtOAoSDIGU=
+github.com/containerd/containerd/v2 v2.1.3 h1:eMD2SLcIQPdMlnlNF6fatlrlRLAeDaiGPGwmRKLZKNs=
+github.com/containerd/containerd/v2 v2.1.3/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM=
 github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
 github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -97,6 +97,8 @@ github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++
 github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
 github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/crazy-max/buildkit v0.7.1-0.20250628122957-0cbe82310f5c h1:BTTtSoqI/XI6M+HE6H3N8T06j4vbDQmXBjsiDZT4f5A=
+github.com/crazy-max/buildkit v0.7.1-0.20250628122957-0cbe82310f5c/go.mod h1:crDmHkHBiiinzUxrpifvvjhkaf+kqa17XqT/B/UgIak=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
 github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
@@ -254,8 +256,6 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX
 github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/buildkit v0.23.0-rc1.0.20250618182037-9b91d20367db h1:ZzrDuG9G1A/RwJvuogNplxCEKsIUQh1CqEnqbOGFgKE=
-github.com/moby/buildkit v0.23.0-rc1.0.20250618182037-9b91d20367db/go.mod h1:v5jMDvQgUyidk3wu3NvVAAd5JJo83nfet9Gf/o0+EAQ=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ=

tests/helpers/k3s.go

@@ -0,0 +1,153 @@
+package helpers
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+const (
+	k3sBin     = "k3s"
+	kubeCtlBin = "kubectl"
+)
+
+func NewK3sServer(cfg *integration.BackendConfig) (kubeConfig string, cl func() error, err error) {
+	if _, err := exec.LookPath(k3sBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3sBin)
+	}
+	if _, err := exec.LookPath(kubeCtlBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", kubeCtlBin)
+	}
+
+	deferF := &integration.MultiCloser{}
+	cl = deferF.F()
+
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	cfgfile, err := os.CreateTemp("", "kubeconfig*.yml")
+	if err != nil {
+		return "", nil, err
+	}
+	kubeConfig = cfgfile.Name()
+	deferF.Append(func() error {
+		return os.Remove(cfgfile.Name())
+	})
+
+	k3sDataDir, err := os.MkdirTemp("", "kubedata")
+	if err != nil {
+		return "", nil, err
+	}
+	deferF.Append(func() error {
+		return os.RemoveAll(k3sDataDir)
+	})
+
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", nil, err
+	}
+	_ = l.Close()
+
+	lport := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
+	nodeName := "integrationk3s"
+
+	mirror := cfg.Mirror
+	if rport, ok := strings.CutPrefix(cfg.Mirror, "localhost:"); ok {
+		mirror = fmt.Sprintf("host.k3s.internal:%s", rport)
+	}
+
+	k3sGlobalDir := "/etc/rancher/k3s"
+	if err := os.MkdirAll(k3sGlobalDir, 0755); err != nil {
+		return "", nil, err
+	}
+	if err := os.WriteFile(filepath.Join(k3sGlobalDir, "registries.yaml"), []byte(fmt.Sprintf(`
+mirrors:
+  "docker.io":
+    endpoint:
+      - "%s"
+
+configs:
+  "%s":
+    insecure_skip_verify: true
+`, mirror, mirror)), 0644); err != nil {
+		return "", nil, err
+	}
+
+	stop, err := integration.StartCmd(exec.Command(k3sBin, "server",
+		"--bind-address", "0.0.0.0",
+		"--https-listen-port", lport,
+		"--data-dir", k3sDataDir, // write to /tmp for overlayfs support
+		"--write-kubeconfig", cfgfile.Name(),
+		"--write-kubeconfig-mode", "666",
+		"--node-name", nodeName,
+	), cfg.Logs)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if err = waitK3s(cfg, kubeConfig, nodeName); err != nil {
+		stop()
+		containerdLogs, _ := os.ReadFile(filepath.Join(k3sDataDir, "agent", "containerd", "containerd.log"))
+		if len(containerdLogs) > 0 {
+			containerdConfig, _ := os.ReadFile(filepath.Join(k3sDataDir, "agent", "etc", "containerd", "config.toml"))
+			registries, _ := os.ReadFile(filepath.Join(k3sGlobalDir, "registries.yaml"))
+			return "", nil, errors.Wrapf(err, "k3s did not start up: %s\ncontainerd.log: %s\ncontainerd.config.toml: %s\nregistries.yaml: %s", formatLogs(cfg.Logs), containerdLogs, containerdConfig, registries)
+		}
+		return "", nil, errors.Wrapf(err, "k3s did not start up: %s", formatLogs(cfg.Logs))
+	}
+
+	deferF.Append(stop)
+	return
+}
+
+func waitK3s(cfg *integration.BackendConfig, kubeConfig string, nodeName string) error {
+	logbuf := new(bytes.Buffer)
+	defer func() {
+		if logbuf.Len() > 0 {
+			cfg.Logs["waitK3s: "] = logbuf
+		}
+	}()
+
+	boff := backoff.NewExponentialBackOff()
+	boff.InitialInterval = 5 * time.Second
+	boff.MaxInterval = 10 * time.Second
+	boff.MaxElapsedTime = 3 * time.Minute
+
+	if err := backoff.Retry(func() error {
+		cmd := exec.Command(kubeCtlBin, "--kubeconfig", kubeConfig, "wait", "--for=condition=Ready", "node/"+nodeName)
+		out, err := cmd.CombinedOutput()
+		if err == nil && bytes.Contains(out, []byte("condition met")) {
+			return nil
+		}
+		return errors.Wrapf(err, "node is not ready: %s %s", cmd.String(), string(out))
+	}, boff); err != nil {
+		logbuf.WriteString(errors.Unwrap(err).Error())
+		return err
+	}
+
+	return nil
+}
+
+func formatLogs(m map[string]*bytes.Buffer) string {
+	var ss []string
+	for k, b := range m {
+		if b != nil {
+			ss = append(ss, fmt.Sprintf("%q:%s", k, b.String()))
+		}
+	}
+	return strings.Join(ss, ",")
+}

tests/integration_test.go

@@ -16,6 +16,7 @@ func init() {
 		workers.InitDockerContainerWorker()
 	} else {
 		workers.InitRemoteWorker()
+		workers.InitKubernetesWorker()
 	}
 }
 

tests/workers/kubernetes.go

@@ -0,0 +1,100 @@
+package workers
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+
+	"github.com/docker/buildx/tests/helpers"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+func InitKubernetesWorker() {
+	integration.Register(&kubernetesWorker{
+		id: "kubernetes",
+	})
+}
+
+type kubernetesWorker struct {
+	id string
+
+	unsupported []string
+
+	k3sConfig string
+	k3sClose  func() error
+	k3sErr    error
+	k3sOnce   sync.Once
+}
+
+func (w *kubernetesWorker) Name() string {
+	return w.id
+}
+
+func (w *kubernetesWorker) Rootless() bool {
+	return false
+}
+
+func (w *kubernetesWorker) NetNSDetached() bool {
+	return false
+}
+
+func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
+	var err error
+
+	w.k3sOnce.Do(func() {
+		w.k3sConfig, w.k3sClose, w.k3sErr = helpers.NewK3sServer(cfg)
+	})
+	if w.k3sErr != nil {
+		return nil, w.k3sClose, w.k3sErr
+	}
+
+	cfgfile, err := integration.WriteConfig(cfg.DaemonConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer os.RemoveAll(filepath.Dir(cfgfile))
+
+	name := "integration-kubernetes-" + identity.NewID()
+	cmd := exec.Command("buildx", "create",
+		"--bootstrap",
+		"--name="+name,
+		"--config="+cfgfile,
+		"--driver=kubernetes",
+	)
+	cmd.Env = append(
+		os.Environ(),
+		"BUILDX_CONFIG=/tmp/buildx-"+name,
+		"KUBECONFIG="+w.k3sConfig,
+	)
+	if err := cmd.Run(); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
+	}
+
+	cl := func() error {
+		cmd := exec.Command("buildx", "rm", "-f", name)
+		return cmd.Run()
+	}
+
+	return &backend{
+		builder:             name,
+		unsupportedFeatures: w.unsupported,
+	}, cl, nil
+}
+
+func (w *kubernetesWorker) Close() error {
+	if c := w.k3sClose; c != nil {
+		return c()
+	}
+
+	// reset the worker to be ready to go again
+	w.k3sConfig = ""
+	w.k3sClose = nil
+	w.k3sErr = nil
+	w.k3sOnce = sync.Once{}
+
+	return nil
+}