oci-worker: experimental support for rootless mode

Signed-off-by: Akihiro Suda <[email protected]>

README.md

@@ -22,6 +22,7 @@ Key features:
 - Distributable workers
 - Multiple output formats
 - Pluggable architecture
+- Execution without root privileges
 
 
 Read the proposal from https://github.com/moby/moby/issues/32925
@@ -225,6 +226,9 @@ export JAEGER_TRACE=0.0.0.0:6831
 
 During development, BuildKit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.1.0/RUNC.md) for more information.
 
+### Running BuildKit without root privileges
+
+Please refer to `[docs/rootless.md]`(docs/rootless.md).
 
 ### Contributing
 

cmd/buildkitd/main.go

@@ -69,21 +69,37 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "buildkitd"
 	app.Usage = "build daemon"
-
-	app.Flags = []cli.Flag{
+	defaultRoot := appdefaults.Root
+	defaultAddress := appdefaults.Address
+	rootlessUsage := "set all the default options to be compatible with rootless containers"
+	if runningAsUnprivilegedUser() {
+		app.Flags = append(app.Flags, cli.BoolTFlag{
+			Name:  "rootless",
+			Usage: rootlessUsage + " (default: true)",
+		})
+		defaultRoot = appdefaults.UserRoot()
+		defaultAddress = appdefaults.UserAddress()
+		appdefaults.EnsureUserAddressDir()
+	} else {
+		app.Flags = append(app.Flags, cli.BoolFlag{
+			Name:  "rootless",
+			Usage: rootlessUsage,
+		})
+	}
+	app.Flags = append(app.Flags,
 		cli.BoolFlag{
 			Name:  "debug",
 			Usage: "enable debug output in logs",
 		},
 		cli.StringFlag{
 			Name:  "root",
 			Usage: "path to state directory",
-			Value: appdefaults.Root,
+			Value: defaultRoot,
 		},
 		cli.StringSliceFlag{
 			Name:  "addr",
 			Usage: "listening address (socket or tcp)",
-			Value: &cli.StringSlice{appdefaults.Address},
+			Value: &cli.StringSlice{defaultAddress},
 		},
 		cli.StringFlag{
 			Name:  "group",
@@ -107,8 +123,7 @@ func main() {
 			Name:  "tlscacert",
 			Usage: "ca certificate to verify clients",
 		},
-	}
-
+	)
 	app.Flags = append(app.Flags, appFlags...)
 
 	app.Action = func(c *cli.Context) error {

cmd/buildkitd/main_containerd_worker.go

@@ -15,12 +15,7 @@ import (
 )
 
 func init() {
-	registerWorkerInitializer(
-		workerInitializer{
-			fn: containerdWorkerInitializer,
-			// 1 is less preferred than 0 (runcCtor)
-			priority: 1,
-		},
+	flags := []cli.Flag{
 		cli.StringFlag{
 			Name:  "containerd-worker",
 			Usage: "enable containerd workers (true/false/auto)",
@@ -35,6 +30,27 @@ func init() {
 			Name:  "containerd-worker-labels",
 			Usage: "user-specific annotation labels (com.example.foo=bar)",
 		},
+	}
+	n := "containerd-worker-rootless"
+	u := "enable rootless mode"
+	if runningAsUnprivilegedUser() {
+		flags = append(flags, cli.BoolTFlag{
+			Name:  n,
+			Usage: u,
+		})
+	} else {
+		flags = append(flags, cli.BoolFlag{
+			Name:  n,
+			Usage: u,
+		})
+	}
+	registerWorkerInitializer(
+		workerInitializer{
+			fn: containerdWorkerInitializer,
+			// 1 is less preferred than 0 (runcCtor)
+			priority: 1,
+		},
+		flags...,
 	)
 	// TODO(AkihiroSuda): allow using multiple snapshotters. should be useful for some applications that does not work with the default overlay snapshotter. e.g. mysql (docker/for-linux#72)",
 }
@@ -52,6 +68,12 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 	if err != nil {
 		return nil, err
 	}
+	// GlobalBool works for BoolT as well
+	rootless := c.GlobalBool("containerd-worker-rootless") || c.GlobalBool("rootless")
+	if rootless {
+		logrus.Warn("rootless mode is not supported for containerd workers. disabling containerd worker.")
+		return nil, nil
+	}
 	opt, err := containerd.NewWorkerOpt(common.root, socket, ctd.DefaultSnapshotter, labels)
 	if err != nil {
 		return nil, err

cmd/buildkitd/main_linux.go

@@ -0,0 +1,9 @@
+// +build linux
+
+package main
+
+import "github.com/opencontainers/runc/libcontainer/system"
+
+func runningAsUnprivilegedUser() bool {
+	return system.GetParentNSeuid() != 0 || system.RunningInUserNS()
+}

cmd/buildkitd/main_nolinux.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package main
+
+func runningAsUnprivilegedUser() bool {
+	return false
+}

cmd/buildkitd/main_oci_worker.go

@@ -17,11 +17,7 @@ import (
 )
 
 func init() {
-	registerWorkerInitializer(
-		workerInitializer{
-			fn:       ociWorkerInitializer,
-			priority: 0,
-		},
+	flags := []cli.Flag{
 		cli.StringFlag{
 			Name:  "oci-worker",
 			Usage: "enable oci workers (true/false/auto)",
@@ -37,8 +33,28 @@ func init() {
 			// TODO(AkihiroSuda): autodetect overlayfs availability when the value is set to "auto"?
 			Value: "overlayfs",
 		},
+	}
+	n := "oci-worker-rootless"
+	u := "enable rootless mode"
+	if runningAsUnprivilegedUser() {
+		flags = append(flags, cli.BoolTFlag{
+			Name:  n,
+			Usage: u,
+		})
+	} else {
+		flags = append(flags, cli.BoolFlag{
+			Name:  n,
+			Usage: u,
+		})
+	}
+	registerWorkerInitializer(
+		workerInitializer{
+			fn:       ociWorkerInitializer,
+			priority: 0,
+		},
+		flags...,
 	)
-	// TODO: allow multiple oci runtimes and snapshotters
+	// TODO: allow multiple oci runtimes
 }
 
 func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker.Worker, error) {
@@ -57,7 +73,12 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 	if err != nil {
 		return nil, err
 	}
-	opt, err := runc.NewWorkerOpt(common.root, snFactory, labels)
+	// GlobalBool works for BoolT as well
+	rootless := c.GlobalBool("oci-worker-rootless") || c.GlobalBool("rootless")
+	if rootless {
+		logrus.Debugf("running in rootless mode")
+	}
+	opt, err := runc.NewWorkerOpt(common.root, snFactory, rootless, labels)
 	if err != nil {
 		return nil, err
 	}

docs/rootless.md

@@ -0,0 +1,51 @@
+# Rootless mode (Experimental)
+
+Requirements:
+- runc (May 30, 2018) or later
+- Some distros such as Debian and Arch Linux require `echo 1 > /proc/sys/kernel/unprivileged_userns_clone`
+- `newuidmap` and `newgidmap` need to be installed on the host. These commands are provided by the `uidmap` package.
+- `/etc/subuid` and `/etc/subgid` should contain >= 65536 sub-IDs. e.g. `penguin:231072:65536`.
+- To run in a Docker container with non-root `USER`, `docker run --privileged` is still required. See also Jessie's blog: https://blog.jessfraz.com/post/building-container-images-securely-on-kubernetes/
+
+Setting up rootless mode also requires some bothersome steps as follows, but we will soon have automation tool.
+
+## Terminal 1:
+
+```
+$ unshare -U -m
+unshared$ echo $$ > /tmp/pid
+```
+
+Unsharing mountns (and userns) is required for mounting filesystems without real root privileges.
+
+## Terminal 2:
+
+```
+$ id -u
+1001
+$ grep $(whoami) /etc/subuid
+penguin:231072:65536
+$ grep $(whoami) /etc/subgid
+penguin:231072:65536
+$ newuidmap $(cat /tmp/pid) 0 1001 1 1 231072 65536
+$ newgidmap $(cat /tmp/pid) 0 1001 1 1 231072 65536
+```
+
+## Terminal 1:
+
+```
+unshared# buildkitd
+```
+
+* The data dir will be set to `/home/penguin/.local/share/buildkit`
+* The address will be set to `unix:///run/user/1001/buildkit/buildkitd.sock`
+* `overlayfs` snapshotter is not supported except Ubuntu-flavored kernel: http://kernel.ubuntu.com/git/ubuntu/ubuntu-artful.git/commit/fs/overlayfs?h=Ubuntu-4.13.0-25.29&id=0a414bdc3d01f3b61ed86cfe3ce8b63a9240eba7
+* containerd worker is not supported ( pending PR: https://github.com/containerd/containerd/pull/2006 )
+* Network namespace is not used at the moment.
+
+## Terminal 2:
+
+```
+$ go get ./examples/build-using-dockerfile
+$ build-using-dockerfile --buildkit-addr unix:///run/user/1001/buildkit/buildkitd.sock -t foo /path/to/somewhere
+```

executor/runcexecutor/executor.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/containerd/containerd/contrib/seccomp"
@@ -18,22 +21,28 @@ import (
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/libcontainer_specconv"
 	"github.com/moby/buildkit/util/system"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 type Opt struct {
+	// root directory
 	Root              string
 	CommandCandidates []string
+	// without root privileges (has nothing to do with Opt.Root directory)
+	Rootless bool
 }
 
 var defaultCommandCandidates = []string{"buildkit-runc", "runc"}
 
 type runcExecutor struct {
-	runc *runc.Runc
-	root string
-	cmd  string
+	runc     *runc.Runc
+	root     string
+	cmd      string
+	rootless bool
 }
 
 func New(opt Opt) (executor.Executor, error) {
@@ -75,8 +84,9 @@ func New(opt Opt) (executor.Executor, error) {
 	}
 
 	w := &runcExecutor{
-		runc: runtime,
-		root: root,
+		runc:     runtime,
+		root:     root,
+		rootless: opt.Rootless,
 	}
 	return w, nil
 }
@@ -156,6 +166,19 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 		return errors.Wrapf(err, "failed to create working directory %s", newp)
 	}
 
+	if w.rootless {
+		specconv.ToRootless(spec, &specconv.RootlessOpts{
+			MapSubUIDGID: true,
+		})
+		// TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable
+		spec.Linux.CgroupsPath = ""
+		// TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here
+		// if either SUID or userspace NAT is configured on the host.
+		if err := setOOMScoreAdj(spec); err != nil {
+			return err
+		}
+	}
+
 	if err := json.NewEncoder(f).Encode(spec); err != nil {
 		return err
 	}
@@ -205,3 +228,19 @@ func (s *forwardIO) Stdout() io.ReadCloser {
 func (s *forwardIO) Stderr() io.ReadCloser {
 	return nil
 }
+
+// setOOMScoreAdj comes from https://github.com/genuinetools/img/blob/2fabe60b7dc4623aa392b515e013bbc69ad510ab/executor/runc/executor.go#L182-L192
+func setOOMScoreAdj(spec *specs.Spec) error {
+	// Set the oom_score_adj of our children containers to that of the current process.
+	b, err := ioutil.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return errors.Wrap(err, "failed to read /proc/self/oom_score_adj")
+	}
+	s := strings.TrimSpace(string(b))
+	oom, err := strconv.Atoi(s)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse %s as int", s)
+	}
+	spec.Process.OOMScoreAdj = &oom
+	return nil
+}