oci-worker: experimental support for rootless mode

Signed-off-by: Akihiro Suda <[email protected]>

README.md

@@ -22,6 +22,7 @@ Key features:
 - Distributable workers
 - Multiple output formats
 - Pluggable architecture
+- Execution without root privileges
 
 
 Read the proposal from https://github.com/moby/moby/issues/32925
@@ -225,6 +226,9 @@ export JAEGER_TRACE=0.0.0.0:6831
 
 During development, BuildKit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.1.0/RUNC.md) for more information.
 
+### Running BuildKit without root privileges
+
+Please refer to `[docs/rootless.md]`(docs/rootless.md).
 
 ### Contributing
 

cmd/buildkitd/main.go

@@ -69,21 +69,37 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "buildkitd"
 	app.Usage = "build daemon"
-
-	app.Flags = []cli.Flag{
+	defaultRoot := appdefaults.Root
+	defaultAddress := appdefaults.Address
+	rootlessUsage := "set all the default options to be compatible with rootless containers"
+	if runningAsUnprivilegedUser() {
+		app.Flags = append(app.Flags, cli.BoolTFlag{
+			Name:  "rootless",
+			Usage: rootlessUsage + " (default: true)",
+		})
+		defaultRoot = appdefaults.UserRoot()
+		defaultAddress = appdefaults.UserAddress()
+		appdefaults.EnsureUserAddressDir()
+	} else {
+		app.Flags = append(app.Flags, cli.BoolFlag{
+			Name:  "rootless",
+			Usage: rootlessUsage,
+		})
+	}
+	app.Flags = append(app.Flags,
 		cli.BoolFlag{
 			Name:  "debug",
 			Usage: "enable debug output in logs",
 		},
 		cli.StringFlag{
 			Name:  "root",
 			Usage: "path to state directory",
-			Value: appdefaults.Root,
+			Value: defaultRoot,
 		},
 		cli.StringSliceFlag{
 			Name:  "addr",
 			Usage: "listening address (socket or tcp)",
-			Value: &cli.StringSlice{appdefaults.Address},
+			Value: &cli.StringSlice{defaultAddress},
 		},
 		cli.StringFlag{
 			Name:  "group",
@@ -107,8 +123,7 @@ func main() {
 			Name:  "tlscacert",
 			Usage: "ca certificate to verify clients",
 		},
-	}
-
+	)
 	app.Flags = append(app.Flags, appFlags...)
 
 	app.Action = func(c *cli.Context) error {

cmd/buildkitd/main_containerd_worker.go

@@ -52,6 +52,12 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 	if err != nil {
 		return nil, err
 	}
+	// GlobalBool works for BoolT as well
+	rootless := c.GlobalBool("rootless")
+	if rootless {
+		logrus.Warn("rootless mode is not supported for containerd workers. disabling containerd worker.")
+		return nil, nil
+	}
 	opt, err := containerd.NewWorkerOpt(common.root, socket, ctd.DefaultSnapshotter, labels)
 	if err != nil {
 		return nil, err

cmd/buildkitd/main_linux.go

@@ -0,0 +1,9 @@
+// +build linux
+
+package main
+
+import "github.com/opencontainers/runc/libcontainer/system"
+
+func runningAsUnprivilegedUser() bool {
+	return system.GetParentNSeuid() != 0 || system.RunningInUserNS()
+}

cmd/buildkitd/main_nolinux.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package main
+
+func runningAsUnprivilegedUser() bool {
+	return false
+}

cmd/buildkitd/main_oci_worker.go

@@ -17,11 +17,7 @@ import (
 )
 
 func init() {
-	registerWorkerInitializer(
-		workerInitializer{
-			fn:       ociWorkerInitializer,
-			priority: 0,
-		},
+	flags := []cli.Flag{
 		cli.StringFlag{
 			Name:  "oci-worker",
 			Usage: "enable oci workers (true/false/auto)",
@@ -34,11 +30,30 @@ func init() {
 		cli.StringFlag{
 			Name:  "oci-worker-snapshotter",
 			Usage: "name of snapshotter (overlayfs or native)",
-			// TODO(AkihiroSuda): autodetect overlayfs availability when the value is set to "auto"?
-			Value: "overlayfs",
+			Value: "auto",
+		},
+	}
+	n := "oci-worker-rootless"
+	u := "enable rootless mode"
+	if runningAsUnprivilegedUser() {
+		flags = append(flags, cli.BoolTFlag{
+			Name:  n,
+			Usage: u,
+		})
+	} else {
+		flags = append(flags, cli.BoolFlag{
+			Name:  n,
+			Usage: u,
+		})
+	}
+	registerWorkerInitializer(
+		workerInitializer{
+			fn:       ociWorkerInitializer,
+			priority: 0,
 		},
+		flags...,
 	)
-	// TODO: allow multiple oci runtimes and snapshotters
+	// TODO: allow multiple oci runtimes
 }
 
 func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker.Worker, error) {
@@ -57,7 +72,12 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 	if err != nil {
 		return nil, err
 	}
-	opt, err := runc.NewWorkerOpt(common.root, snFactory, labels)
+	// GlobalBool works for BoolT as well
+	rootless := c.GlobalBool("oci-worker-rootless") || c.GlobalBool("rootless")
+	if rootless {
+		logrus.Debugf("running in rootless mode")
+	}
+	opt, err := runc.NewWorkerOpt(common.root, snFactory, rootless, labels)
 	if err != nil {
 		return nil, err
 	}
@@ -75,6 +95,16 @@ func snapshotterFactory(name string) (runc.SnapshotterFactory, error) {
 	}
 	var err error
 	switch name {
+	case "auto":
+		snFactory.New = func(root string) (ctdsnapshot.Snapshotter, error) {
+			err := overlay.Supported(root)
+			if err == nil {
+				logrus.Debug("auto snapshotter: using overlayfs")
+				return overlay.NewSnapshotter(root)
+			}
+			logrus.Debugf("auto snapshotter: using native for %s: %v", root, err)
+			return native.NewSnapshotter(root)
+		}
 	case "native":
 		snFactory.New = native.NewSnapshotter
 	case "overlayfs": // not "overlay", for consistency with containerd snapshotter plugin ID.

docs/rootless.md

@@ -0,0 +1,51 @@
+# Rootless mode (Experimental)
+
+Requirements:
+- runc (May 30, 2018) or later
+- Some distros such as Debian and Arch Linux require `echo 1 > /proc/sys/kernel/unprivileged_userns_clone`
+- `newuidmap` and `newgidmap` need to be installed on the host. These commands are provided by the `uidmap` package.
+- `/etc/subuid` and `/etc/subgid` should contain >= 65536 sub-IDs. e.g. `penguin:231072:65536`.
+- To run in a Docker container with non-root `USER`, `docker run --privileged` is still required. See also Jessie's blog: https://blog.jessfraz.com/post/building-container-images-securely-on-kubernetes/
+
+Setting up rootless mode also requires some bothersome steps as follows, but we will soon have automation tool.
+
+## Terminal 1:
+
+```
+$ unshare -U -m
+unshared$ echo $$ > /tmp/pid
+```
+
+Unsharing mountns (and userns) is required for mounting filesystems without real root privileges.
+
+## Terminal 2:
+
+```
+$ id -u
+1001
+$ grep $(whoami) /etc/subuid
+penguin:231072:65536
+$ grep $(whoami) /etc/subgid
+penguin:231072:65536
+$ newuidmap $(cat /tmp/pid) 0 1001 1 1 231072 65536
+$ newgidmap $(cat /tmp/pid) 0 1001 1 1 231072 65536
+```
+
+## Terminal 1:
+
+```
+unshared# buildkitd
+```
+
+* The data dir will be set to `/home/penguin/.local/share/buildkit`
+* The address will be set to `unix:///run/user/1001/buildkit/buildkitd.sock`
+* `overlayfs` snapshotter is not supported except Ubuntu-flavored kernel: http://kernel.ubuntu.com/git/ubuntu/ubuntu-artful.git/commit/fs/overlayfs?h=Ubuntu-4.13.0-25.29&id=0a414bdc3d01f3b61ed86cfe3ce8b63a9240eba7
+* containerd worker is not supported ( pending PR: https://github.com/containerd/containerd/pull/2006 )
+* Network namespace is not used at the moment.
+
+## Terminal 2:
+
+```
+$ go get ./examples/build-using-dockerfile
+$ build-using-dockerfile --buildkit-addr unix:///run/user/1001/buildkit/buildkitd.sock -t foo /path/to/somewhere
+```

executor/runcexecutor/executor.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/containerd/containerd/contrib/seccomp"
@@ -18,22 +21,28 @@ import (
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/libcontainer_specconv"
 	"github.com/moby/buildkit/util/system"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 type Opt struct {
+	// root directory
 	Root              string
 	CommandCandidates []string
+	// without root privileges (has nothing to do with Opt.Root directory)
+	Rootless bool
 }
 
 var defaultCommandCandidates = []string{"buildkit-runc", "runc"}
 
 type runcExecutor struct {
-	runc *runc.Runc
-	root string
-	cmd  string
+	runc     *runc.Runc
+	root     string
+	cmd      string
+	rootless bool
 }
 
 func New(opt Opt) (executor.Executor, error) {
@@ -75,8 +84,9 @@ func New(opt Opt) (executor.Executor, error) {
 	}
 
 	w := &runcExecutor{
-		runc: runtime,
-		root: root,
+		runc:     runtime,
+		root:     root,
+		rootless: opt.Rootless,
 	}
 	return w, nil
 }
@@ -156,6 +166,19 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 		return errors.Wrapf(err, "failed to create working directory %s", newp)
 	}
 
+	if w.rootless {
+		specconv.ToRootless(spec, &specconv.RootlessOpts{
+			MapSubUIDGID: true,
+		})
+		// TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable
+		spec.Linux.CgroupsPath = ""
+		// TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here
+		// if either SUID or userspace NAT is configured on the host.
+		if err := setOOMScoreAdj(spec); err != nil {
+			return err
+		}
+	}
+
 	if err := json.NewEncoder(f).Encode(spec); err != nil {
 		return err
 	}
@@ -205,3 +228,19 @@ func (s *forwardIO) Stdout() io.ReadCloser {
 func (s *forwardIO) Stderr() io.ReadCloser {
 	return nil
 }
+
+// setOOMScoreAdj comes from https://github.com/genuinetools/img/blob/2fabe60b7dc4623aa392b515e013bbc69ad510ab/executor/runc/executor.go#L182-L192
+func setOOMScoreAdj(spec *specs.Spec) error {
+	// Set the oom_score_adj of our children containers to that of the current process.
+	b, err := ioutil.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return errors.Wrap(err, "failed to read /proc/self/oom_score_adj")
+	}
+	s := strings.TrimSpace(string(b))
+	oom, err := strconv.Atoi(s)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse %s as int", s)
+	}
+	spec.Process.OOMScoreAdj = &oom
+	return nil
+}

util/appdefaults/appdefaults_unix.go

@@ -2,7 +2,54 @@
 
 package appdefaults
 
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
+
 const (
 	Address = "unix:///run/buildkit/buildkitd.sock"
 	Root    = "/var/lib/buildkit"
 )
+
+// UserAddress typically returns /run/user/$UID/buildkit/buildkitd.sock
+func UserAddress() string {
+	//  pam_systemd sets XDG_RUNTIME_DIR but not other dirs.
+	xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR")
+	if xdgRuntimeDir != "" {
+		dirs := strings.Split(xdgRuntimeDir, ":")
+		return "unix://" + filepath.Join(dirs[0], "buildkit", "buildkitd.sock")
+	}
+	return Address
+}
+
+// EnsureUserAddressDir sets sticky bit on XDG_RUNTIME_DIR if XDG_RUNTIME_DIR is set.
+// See https://github.com/opencontainers/runc/issues/1694
+func EnsureUserAddressDir() error {
+	xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR")
+	if xdgRuntimeDir != "" {
+		dirs := strings.Split(xdgRuntimeDir, ":")
+		dir := filepath.Join(dirs[0], "buildkit")
+		if err := os.MkdirAll(dir, 0700); err != nil {
+			return err
+		}
+		return os.Chmod(dir, 0700|os.ModeSticky)
+	}
+	return nil
+}
+
+// UserRoot typically returns /home/$USER/.local/share/buildkit
+func UserRoot() string {
+	//  pam_systemd sets XDG_RUNTIME_DIR but not other dirs.
+	xdgDataHome := os.Getenv("XDG_DATA_HOME")
+	if xdgDataHome != "" {
+		dirs := strings.Split(xdgDataHome, ":")
+		return filepath.Join(dirs[0], "buildkit")
+	}
+	home := os.Getenv("HOME")
+	if home != "" {
+		return filepath.Join(home, ".local", "share", "buildkit")
+	}
+	return Root
+}

util/appdefaults/appdefaults_windows.go

@@ -4,3 +4,15 @@ const (
 	Address = "npipe:////./pipe/buildkitd"
 	Root    = ".buildstate"
 )
+
+func UserAddress() string {
+	return Address
+}
+
+func EnsureUserAddressDir() error {
+	return nil
+}
+
+func UserRoot() string {
+	return Root
+}