From a8d533d83a3716409c172d12a8c1f09e6dba0cba Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sun, 10 Dec 2023 11:58:22 +0100 Subject: [PATCH] vm: inotify: monitor all containerd namespaces for volumes (#923) * vm: inotify: monitor all containerd namespaces for volumes * docker: ensure user added to group, reduce wait interval --- daemon/process/inotify/volumes.go | 142 ++++++++++++++++--------- environment/container/docker/docker.go | 16 ++- 2 files changed, 102 insertions(+), 56 deletions(-) diff --git a/daemon/process/inotify/volumes.go b/daemon/process/inotify/volumes.go index 47d3fefcf..b07ec39c9 100644 --- a/daemon/process/inotify/volumes.go +++ b/daemon/process/inotify/volumes.go @@ -19,70 +19,37 @@ func (f *inotifyProcess) monitorContainerVolumes(ctx context.Context, c chan<- [ if f.runtime == "" { return fmt.Errorf("empty runtime") } - runtimeCmd := docker.Name - if f.runtime == containerd.Name { - runtimeCmd = "nerdctl" - } fetch := func() ([]string, error) { - // fetch all containers - var containers []string - { - out, err := f.guest.RunOutput(runtimeCmd, "ps", "-q") + var vols []string + + // docker + if f.runtime != containerd.Name { + vols, err := f.fetchVolumes(docker.Name) if err != nil { - return nil, fmt.Errorf("error listing containers: %w", err) - } - containers = strings.Fields(out) - if len(containers) == 0 { - return nil, nil + return nil, fmt.Errorf("error fetching docker volumes: %w", err) } + return vols, nil } - log.Tracef("found containers %+v", containers) - - // fetch volumes - var resp []struct { - Mounts []struct { - Source string `json:"Source"` - } `json:"Mounts"` + // containerd + var namespaces []string + out, err := f.guest.RunOutput("sudo", "nerdctl", "namespace", "list", "-q") + if err != nil { + return nil, fmt.Errorf("error retrieving containerd namespaces: %w", err) } - { - args := []string{runtimeCmd, "inspect"} - args = append(args, containers...) - - var buf bytes.Buffer - if err := f.guest.RunWith(nil, &buf, args...); err != nil { - return nil, fmt.Errorf("error inspecting containers: %w", err) - } - if err := json.NewDecoder(&buf).Decode(&resp); err != nil { - return nil, fmt.Errorf("error decoding docker response") - } + if out != "" { + namespaces = strings.Fields(out) } - // process and discard redundant volumes - vols := []string{} - { - shouldMount := func(child string) bool { - // ignore all invalid directories. - // i.e. directories not within the mounted VM directories - for _, parent := range f.vmVols { - if strings.HasPrefix(child, parent) { - return true - } - } - return false + for _, ns := range namespaces { + v, err := f.fetchVolumes("sudo", "nerdctl", "--namespace", ns) + if err != nil { + return nil, fmt.Errorf("error retrieving containerd volumes: %w", err) } - - for _, r := range resp { - for _, mount := range r.Mounts { - if shouldMount(mount.Source) { - vols = append(vols, mount.Source) - } - } + if len(v) > 0 { + vols = append(vols, v...) } - - vols = omitChildrenDirectories(vols) - log.Tracef("found volumes %+v", vols) } return vols, nil @@ -110,6 +77,75 @@ func (f *inotifyProcess) monitorContainerVolumes(ctx context.Context, c chan<- [ return nil } +func (f *inotifyProcess) fetchVolumes(cmdArgs ...string) ([]string, error) { + log := f.log + + // fetch all containers + var containers []string + { + args := append([]string{}, cmdArgs...) + args = append(args, "ps", "-q") + out, err := f.guest.RunOutput(args...) + if err != nil { + return nil, fmt.Errorf("error listing containers: %w", err) + } + containers = strings.Fields(out) + if len(containers) == 0 { + return nil, nil + } + } + + log.Tracef("found containers %+v", containers) + + // fetch volumes + var resp []struct { + Mounts []struct { + Source string `json:"Source"` + } `json:"Mounts"` + } + { + args := append([]string{}, cmdArgs...) + args = append(args, "inspect") + args = append(args, containers...) + + var buf bytes.Buffer + if err := f.guest.RunWith(nil, &buf, args...); err != nil { + return nil, fmt.Errorf("error inspecting containers: %w", err) + } + if err := json.NewDecoder(&buf).Decode(&resp); err != nil { + return nil, fmt.Errorf("error decoding docker response") + } + } + + // process and discard redundant volumes + vols := []string{} + { + shouldMount := func(child string) bool { + // ignore all invalid directories. + // i.e. directories not within the mounted VM directories + for _, parent := range f.vmVols { + if strings.HasPrefix(child, parent) { + return true + } + } + return false + } + + for _, r := range resp { + for _, mount := range r.Mounts { + if shouldMount(mount.Source) { + vols = append(vols, mount.Source) + } + } + } + + vols = omitChildrenDirectories(vols) + log.Tracef("found volumes %+v", vols) + } + + return vols, nil +} + func omitChildrenDirectories(dirs []string) []string { sort.Strings(dirs) // sort to put the parent directories first diff --git a/environment/container/docker/docker.go b/environment/container/docker/docker.go index a966a868f..fc3e2d7fb 100644 --- a/environment/container/docker/docker.go +++ b/environment/container/docker/docker.go @@ -69,15 +69,25 @@ func (d dockerRuntime) Start(ctx context.Context) error { // TODO: interval is high due to 0.6.3->0.6.4 docker-ce package transition // to ensure startup is successful - a.Retry("", time.Second*5, 24, func(int) error { + a.Retry("", time.Second, 120, func(int) error { return d.guest.RunQuiet("sudo", "service", "docker", "start") }) - // service startup takes few seconds, retry at most 5 times before giving up. - a.Retry("", time.Second*5, 12, func(int) error { + // service startup takes few seconds, retry for a minute before giving up. + a.Retry("", time.Second, 60, func(int) error { return d.guest.RunQuiet("sudo", "docker", "info") }) + // ensure docker is accessible without root + // otherwise, restart to ensure user is added to docker group + a.Add(func() error { + if err := d.guest.RunQuiet("docker", "info"); err == nil { + return nil + } + ctx := context.WithValue(ctx, cli.CtxKeyQuiet, true) + return d.guest.Restart(ctx) + }) + return a.Exec() }