diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 62a1286bb..4e12fd095 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -103,8 +103,9 @@ jobs: # - Ensure manifests work with the latest versions even with no manifest change # (compared to helm charts, manifests cannot easily template changes based on versions) # Helm charts are _trailing_ releases, while manifests are done during development. - e2e-manifests: - name: End-to-End test with kured with code and manifests from HEAD + # This test uses the "command" reboot-method. + e2e-manifests-command: + name: End-to-End test with kured with code and manifests from HEAD (command) runs-on: ubuntu-latest strategy: fail-fast: false @@ -181,6 +182,91 @@ jobs: ./tests/kind/follow-coordinated-reboot.sh + # This ensures the latest code works with the manifests built from tree. + # It is useful for two things: + # - Test manifests changes (obviously), ensuring they don't break existing clusters + # - Ensure manifests work with the latest versions even with no manifest change + # (compared to helm charts, manifests cannot easily template changes based on versions) + # Helm charts are _trailing_ releases, while manifests are done during development. + # This test uses the "signal" reboot-method. + e2e-manifests-signal: + name: End-to-End test with kured with code and manifests from HEAD (signal) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + kubernetes: + - "1.26" + - "1.27" + - "1.28" + steps: + - uses: actions/checkout@v3 + - name: Ensure go version + uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + check-latest: true + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Setup GoReleaser + run: make bootstrap-tools + - name: Find current tag version + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + id: tags + - name: Build artifacts + run: | + VERSION="${{ steps.tags.outputs.sha_short }}" make image + VERSION="${{ steps.tags.outputs.sha_short }}" make manifest + + - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions + run: | + sudo bash << EOF + cp /etc/docker/daemon.json /etc/docker/daemon.json.old + echo '{}' > /etc/docker/daemon.json + systemctl restart docker || journalctl --no-pager -n 500 + systemctl status docker + EOF + + # Default name for helm/kind-action kind clusters is "chart-testing" + - name: Create kind cluster with 5 nodes + uses: helm/kind-action@v1.8.0 + with: + config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + version: v0.14.0 + + - name: Preload previously built images onto kind cluster + run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing + + - name: Do not wait for an hour before detecting the rebootSentinel + run: | + sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml + + - name: Install kured with kubectl + run: | + kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds-signal.yaml + + - name: Ensure kured is ready + uses: nick-invision/retry@v2.8.3 + with: + timeout_minutes: 10 + max_attempts: 10 + retry_wait_seconds: 60 + # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size + command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" + + - name: Create reboot sentinel files + run: | + ./tests/kind/create-reboot-sentinels.sh + + - name: Follow reboot until success + env: + DEBUG: true + run: | + ./tests/kind/follow-coordinated-reboot.sh + + # This ensures the latest code works with the manifests built from tree. # It is useful for two things: diff --git a/Makefile b/Makefile index 7ba912db6..372cd545b 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ minikube-publish: image manifest: sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds.yaml + sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds-signal.yaml echo "Please generate combined manifest if necessary" test: diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7440d3f46..3ad1cbee2 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -33,8 +33,10 @@ import ( "github.com/kubereboot/kured/pkg/alerts" "github.com/kubereboot/kured/pkg/daemonsetlock" "github.com/kubereboot/kured/pkg/delaytick" + "github.com/kubereboot/kured/pkg/reboot" "github.com/kubereboot/kured/pkg/taints" "github.com/kubereboot/kured/pkg/timewindow" + "github.com/kubereboot/kured/pkg/util" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -47,6 +49,7 @@ var ( drainDelay time.Duration drainTimeout time.Duration rebootDelay time.Duration + rebootMethod string period time.Duration metricsHost string metricsPort int @@ -74,6 +77,7 @@ var ( messageTemplateUncordon string podSelectors []string rebootCommand string + rebootSignal int logFormat string preRebootNodeLabels []string postRebootNodeLabels []string @@ -103,6 +107,13 @@ const ( KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed" // EnvPrefix The environment variable prefix of all environment variables bound to our command line flags. EnvPrefix = "KURED" + + // MethodCommand is used as "--reboot-method" value when rebooting with the configured "--reboot-command" + MethodCommand = "command" + // MethodSignal is used as "--reboot-method" value when rebooting with a SIGRTMIN+5 signal. + MethodSignal = "signal" + + sigTrminPlus5 = 34 + 5 ) func init() { @@ -146,6 +157,8 @@ func NewRootCommand() *cobra.Command { "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&rebootDelay, "reboot-delay", 0, "delay reboot for this duration (default: 0, disabled)") + rootCmd.PersistentFlags().StringVar(&rebootMethod, "reboot-method", "command", + "method to use for reboots. Available: command") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -176,6 +189,8 @@ func NewRootCommand() *cobra.Command { "command to run when a reboot is required") rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 1, "amount of nodes to concurrently reboot. Defaults to 1") + rootCmd.PersistentFlags().IntVar(&rebootSignal, "reboot-signal", sigTrminPlus5, + "signal to use for reboot, SIGRTMIN+5 by default.") rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "", "slack hook URL for reboot notifications [deprecated in favor of --notify-url]") @@ -299,22 +314,6 @@ func flagToEnvVar(flag string) string { return fmt.Sprintf("%s_%s", EnvPrefix, envVarSuffix) } -// newCommand creates a new Command with stdout/stderr wired to our standard logger -func newCommand(name string, arg ...string) *exec.Cmd { - cmd := exec.Command(name, arg...) - cmd.Stdout = log.NewEntry(log.StandardLogger()). - WithField("cmd", cmd.Args[0]). - WithField("std", "out"). - WriterLevel(log.InfoLevel) - - cmd.Stderr = log.NewEntry(log.StandardLogger()). - WithField("cmd", cmd.Args[0]). - WithField("std", "err"). - WriterLevel(log.WarnLevel) - - return cmd -} - // buildHostCommand writes a new command to run in the host namespace // Rancher based need different pid func buildHostCommand(pid int, command []string) []string { @@ -327,7 +326,7 @@ func buildHostCommand(pid int, command []string) []string { } func rebootRequired(sentinelCommand []string) bool { - cmd := newCommand(sentinelCommand[0], sentinelCommand[1:]...) + cmd := util.NewCommand(sentinelCommand[0], sentinelCommand[1:]...) if err := cmd.Run(); err != nil { switch err := err.(type) { case *exec.ExitError: @@ -557,20 +556,6 @@ func uncordon(client *kubernetes.Clientset, node *v1.Node) error { return nil } -func invokeReboot(nodeID string, rebootCommand []string) { - log.Infof("Running command: %s for node: %s", rebootCommand, nodeID) - - if notifyURL != "" { - if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil { - log.Warnf("Error notifying: %v", err) - } - } - - if err := newCommand(rebootCommand[0], rebootCommand[1:]...).Run(); err != nil { - log.Fatalf("Error invoking reboot command: %v", err) - } -} - func maintainRebootRequiredMetric(nodeID string, sentinelCommand []string) { for { if rebootRequired(sentinelCommand) { @@ -661,7 +646,7 @@ func updateNodeLabels(client *kubernetes.Clientset, node *v1.Node, labels []stri } } -func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) { +func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) { config, err := rest.InClusterConfig() if err != nil { log.Fatal(err) @@ -805,7 +790,13 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s time.Sleep(rebootDelay) } - invokeReboot(nodeID, rebootCommand) + if notifyURL != "" { + if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil { + log.Warnf("Error notifying: %v", err) + } + } + + booter.Reboot() for { log.Infof("Waiting for reboot") time.Sleep(time.Minute) @@ -872,7 +863,13 @@ func root(cmd *cobra.Command, args []string) { log.Infof("Reboot schedule: %v", window) log.Infof("Reboot check command: %s every %v", sentinelCommand, period) log.Infof("Concurrency: %v", concurrency) - log.Infof("Reboot command: %s", restartCommand) + log.Infof("Reboot method: %s", rebootMethod) + if rebootCommand == MethodCommand { + log.Infof("Reboot command: %s", restartCommand) + } else { + log.Infof("Reboot signal: %v", rebootSignal) + } + if annotateNodes { log.Infof("Will annotate nodes during kured reboot operations") } @@ -880,10 +877,24 @@ func root(cmd *cobra.Command, args []string) { // To run those commands as it was the host, we'll use nsenter // Relies on hostPID:true and privileged:true to enter host mount space // PID set to 1, until we have a better discovery mechanism. - hostSentinelCommand := buildHostCommand(1, sentinelCommand) hostRestartCommand := buildHostCommand(1, restartCommand) - go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL, lockReleaseDelay) + // Only wrap sentinel-command with nsenter, if a custom-command was configured, otherwise use the host-path mount + hostSentinelCommand := sentinelCommand + if rebootSentinelCommand != "" { + hostSentinelCommand = buildHostCommand(1, sentinelCommand) + } + + var booter reboot.Reboot + if rebootMethod == MethodCommand { + booter = reboot.NewCommandReboot(nodeID, hostRestartCommand) + } else if rebootMethod == MethodSignal { + booter = reboot.NewSignalReboot(nodeID, rebootSignal) + } else { + log.Fatalf("Invalid reboot-method configured: %s", rebootMethod) + } + + go rebootAsRequired(nodeID, booter, hostSentinelCommand, window, lockTTL, lockReleaseDelay) go maintainRebootRequiredMetric(nodeID, hostSentinelCommand) http.Handle("/metrics", promhttp.Handler()) diff --git a/kured-ds-signal.yaml b/kured-ds-signal.yaml new file mode 100644 index 000000000..9fe000cad --- /dev/null +++ b/kured-ds-signal.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kured + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kured # Must match `--ds-name` + namespace: kube-system # Must match `--ds-namespace` +spec: + selector: + matchLabels: + name: kured + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: kured + spec: + serviceAccountName: kured + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + hostPID: true # Facilitate entering the host mount namespace via init + restartPolicy: Always + volumes: + - name: sentinel + hostPath: + path: /var/run + type: Directory + containers: + - name: kured + # If you find yourself here wondering why there is no + # :latest tag on Docker Hub,see the FAQ in the README + image: ghcr.io/kubereboot/kured:1.13.2 + imagePullPolicy: IfNotPresent + securityContext: + privileged: false # Give permission to nsenter /proc/1/ns/mnt + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["*"] + add: ["CAP_KILL"] + ports: + - containerPort: 8080 + name: metrics + env: + # Pass in the name of the node on which this pod is scheduled + # for use with drain/uncordon operations and lock acquisition + - name: KURED_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /sentinel + name: sentinel + readOnly: true + command: + - /usr/bin/kured + - --reboot-sentinel=/sentinel/reboot-required + - --reboot-method=signal +# - --reboot-signal=39 +# - --force-reboot=false +# - --drain-grace-period=-1 +# - --skip-wait-for-delete-timeout=0 +# - --drain-timeout=0 +# - --period=1h +# - --ds-namespace=kube-system +# - --ds-name=kured +# - --lock-annotation=weave.works/kured-node-lock +# - --lock-ttl=0 +# - --prometheus-url=http://prometheus.monitoring.svc.cluster.local +# - --alert-filter-regexp=^RebootRequired$ +# - --alert-firing-only=false +# - --prefer-no-schedule-taint="" +# - --reboot-sentinel-command="" +# - --slack-hook-url=https://hooks.slack.com/... +# - --slack-username=prod +# - --slack-channel=alerting +# - --notify-url="" # See also shoutrrr url format +# - --message-template-drain=Draining node %s +# - --message-template-reboot=Rebooting node %s +# - --message-template-uncordon=Node %s rebooted & uncordoned successfully! +# - --blocking-pod-selector=runtime=long,cost=expensive +# - --blocking-pod-selector=name=temperamental +# - --blocking-pod-selector=... +# - --reboot-days=sun,mon,tue,wed,thu,fri,sat +# - --reboot-delay=90s +# - --start-time=0:00 +# - --end-time=23:59:59 +# - --time-zone=UTC +# - --annotate-nodes=false +# - --lock-release-delay=30m +# - --log-format=text diff --git a/kured-ds.yaml b/kured-ds.yaml index 77bd67d0b..8bc7e18f8 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -29,6 +29,11 @@ spec: effect: NoSchedule hostPID: true # Facilitate entering the host mount namespace via init restartPolicy: Always + volumes: + - name: sentinel + hostPath: + path: /var/run + type: Directory containers: - name: kured # If you find yourself here wondering why there is no @@ -48,8 +53,13 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + volumeMounts: + - mountPath: /sentinel + name: sentinel + readOnly: true command: - /usr/bin/kured + - --reboot-sentinel=/sentinel/reboot-required # - --force-reboot=false # - --drain-grace-period=-1 # - --skip-wait-for-delete-timeout=0 @@ -65,9 +75,10 @@ spec: # - --alert-filter-regexp=^RebootRequired$ # - --alert-filter-match-only=false # - --alert-firing-only=false -# - --reboot-sentinel=/var/run/reboot-required # - --prefer-no-schedule-taint="" # - --reboot-sentinel-command="" +# - --reboot-method=command +# - --reboot-signal=39 # - --slack-hook-url=https://hooks.slack.com/... # - --slack-username=prod # - --slack-channel=alerting diff --git a/pkg/reboot/command.go b/pkg/reboot/command.go new file mode 100644 index 000000000..1522d2a86 --- /dev/null +++ b/pkg/reboot/command.go @@ -0,0 +1,25 @@ +package reboot + +import ( + "github.com/kubereboot/kured/pkg/util" + log "github.com/sirupsen/logrus" +) + +// CommandRebootMethod holds context-information for a command reboot. +type CommandRebootMethod struct { + nodeID string + rebootCommand []string +} + +// NewCommandReboot creates a new command-rebooter which needs full privileges on the host. +func NewCommandReboot(nodeID string, rebootCommand []string) *CommandRebootMethod { + return &CommandRebootMethod{nodeID: nodeID, rebootCommand: rebootCommand} +} + +// Reboot triggers the command-reboot. +func (c *CommandRebootMethod) Reboot() { + log.Infof("Running command: %s for node: %s", c.rebootCommand, c.nodeID) + if err := util.NewCommand(c.rebootCommand[0], c.rebootCommand[1:]...).Run(); err != nil { + log.Fatalf("Error invoking reboot command: %v", err) + } +} diff --git a/pkg/reboot/reboot.go b/pkg/reboot/reboot.go new file mode 100644 index 000000000..83d788ecf --- /dev/null +++ b/pkg/reboot/reboot.go @@ -0,0 +1,6 @@ +package reboot + +// Reboot interface defines the Reboot function to be implemented. +type Reboot interface { + Reboot() +} diff --git a/pkg/reboot/signal.go b/pkg/reboot/signal.go new file mode 100644 index 000000000..245018c2e --- /dev/null +++ b/pkg/reboot/signal.go @@ -0,0 +1,34 @@ +package reboot + +import ( + "os" + "syscall" + + log "github.com/sirupsen/logrus" +) + +// SignalRebootMethod holds context-information for a signal reboot. +type SignalRebootMethod struct { + nodeID string + signal int +} + +// NewSignalReboot creates a new signal-rebooter which can run unprivileged. +func NewSignalReboot(nodeID string, signal int) *SignalRebootMethod { + return &SignalRebootMethod{nodeID: nodeID, signal: signal} +} + +// Reboot triggers the signal-reboot. +func (c *SignalRebootMethod) Reboot() { + log.Infof("Emit reboot-signal for node: %s", c.nodeID) + + process, err := os.FindProcess(1) + if err != nil { + log.Fatalf("There was no systemd process found: %v", err) + } + + err = process.Signal(syscall.Signal(c.signal)) + if err != nil { + log.Fatalf("Signal of SIGRTMIN+5 failed: %v", err) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..d32f9d1c2 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,23 @@ +package util + +import ( + "os/exec" + + log "github.com/sirupsen/logrus" +) + +// NewCommand creates a new Command with stdout/stderr wired to our standard logger +func NewCommand(name string, arg ...string) *exec.Cmd { + cmd := exec.Command(name, arg...) + cmd.Stdout = log.NewEntry(log.StandardLogger()). + WithField("cmd", cmd.Args[0]). + WithField("std", "out"). + WriterLevel(log.InfoLevel) + + cmd.Stderr = log.NewEntry(log.StandardLogger()). + WithField("cmd", cmd.Args[0]). + WithField("std", "err"). + WriterLevel(log.WarnLevel) + + return cmd +}