-
Notifications
You must be signed in to change notification settings - Fork 211
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: sentinel-command without nsenter by default Signed-off-by: Christian Kotzbauer <[email protected]> * fix: no readonly mount Signed-off-by: Christian Kotzbauer <[email protected]> * fix: mount at different folder Signed-off-by: Christian Kotzbauer <[email protected]> * feat: add signal-reboot Signed-off-by: Christian Kotzbauer <[email protected]> * feat: make signal configurable and add tests Signed-off-by: Christian Kotzbauer <[email protected]> * build: rename job Signed-off-by: Christian Kotzbauer <[email protected]> * cleanup: linter Signed-off-by: Christian Kotzbauer <[email protected]> * build: also adjust signal manifest Signed-off-by: Christian Kotzbauer <[email protected]> * test: add e2e-tests Signed-off-by: Christian Kotzbauer <[email protected]> * fix: small code restructure Signed-off-by: Christian Kotzbauer <[email protected]> * fix: adjust version-range Signed-off-by: Christian Kotzbauer <[email protected]> --------- Signed-off-by: Christian Kotzbauer <[email protected]>
- Loading branch information
1 parent
fb800aa
commit 87202d8
Showing
9 changed files
with
336 additions
and
39 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -103,8 +103,9 @@ jobs: | |
| # - Ensure manifests work with the latest versions even with no manifest change | ||
| # (compared to helm charts, manifests cannot easily template changes based on versions) | ||
| # Helm charts are _trailing_ releases, while manifests are done during development. | ||
| e2e-manifests: | ||
| name: End-to-End test with kured with code and manifests from HEAD | ||
| # This test uses the "command" reboot-method. | ||
| e2e-manifests-command: | ||
| name: End-to-End test with kured with code and manifests from HEAD (command) | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| fail-fast: false | ||
|
|
@@ -181,6 +182,91 @@ jobs: | |
| ./tests/kind/follow-coordinated-reboot.sh | ||
| # This ensures the latest code works with the manifests built from tree. | ||
| # It is useful for two things: | ||
| # - Test manifests changes (obviously), ensuring they don't break existing clusters | ||
| # - Ensure manifests work with the latest versions even with no manifest change | ||
| # (compared to helm charts, manifests cannot easily template changes based on versions) | ||
| # Helm charts are _trailing_ releases, while manifests are done during development. | ||
| # This test uses the "signal" reboot-method. | ||
| e2e-manifests-signal: | ||
| name: End-to-End test with kured with code and manifests from HEAD (signal) | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| kubernetes: | ||
| - "1.26" | ||
| - "1.27" | ||
| - "1.28" | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Ensure go version | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version-file: 'go.mod' | ||
| check-latest: true | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v2 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v2 | ||
| - name: Setup GoReleaser | ||
| run: make bootstrap-tools | ||
| - name: Find current tag version | ||
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | ||
| id: tags | ||
| - name: Build artifacts | ||
| run: | | ||
| VERSION="${{ steps.tags.outputs.sha_short }}" make image | ||
| VERSION="${{ steps.tags.outputs.sha_short }}" make manifest | ||
| - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions | ||
| run: | | ||
| sudo bash << EOF | ||
| cp /etc/docker/daemon.json /etc/docker/daemon.json.old | ||
| echo '{}' > /etc/docker/daemon.json | ||
| systemctl restart docker || journalctl --no-pager -n 500 | ||
| systemctl status docker | ||
| EOF | ||
| # Default name for helm/kind-action kind clusters is "chart-testing" | ||
| - name: Create kind cluster with 5 nodes | ||
| uses: helm/[email protected] | ||
| with: | ||
| config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml | ||
| version: v0.14.0 | ||
|
|
||
| - name: Preload previously built images onto kind cluster | ||
| run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing | ||
|
|
||
| - name: Do not wait for an hour before detecting the rebootSentinel | ||
| run: | | ||
| sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml | ||
| - name: Install kured with kubectl | ||
| run: | | ||
| kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds-signal.yaml | ||
| - name: Ensure kured is ready | ||
| uses: nick-invision/[email protected] | ||
| with: | ||
| timeout_minutes: 10 | ||
| max_attempts: 10 | ||
| retry_wait_seconds: 60 | ||
| # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size | ||
| command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" | ||
|
|
||
| - name: Create reboot sentinel files | ||
| run: | | ||
| ./tests/kind/create-reboot-sentinels.sh | ||
| - name: Follow reboot until success | ||
| env: | ||
| DEBUG: true | ||
| run: | | ||
| ./tests/kind/follow-coordinated-reboot.sh | ||
| # This ensures the latest code works with the manifests built from tree. | ||
| # It is useful for two things: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| --- | ||
| apiVersion: v1 | ||
| kind: ServiceAccount | ||
| metadata: | ||
| name: kured | ||
| namespace: kube-system | ||
| --- | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: kured # Must match `--ds-name` | ||
| namespace: kube-system # Must match `--ds-namespace` | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| name: kured | ||
| updateStrategy: | ||
| type: RollingUpdate | ||
| template: | ||
| metadata: | ||
| labels: | ||
| name: kured | ||
| spec: | ||
| serviceAccountName: kured | ||
| tolerations: | ||
| - key: node-role.kubernetes.io/control-plane | ||
| effect: NoSchedule | ||
| - key: node-role.kubernetes.io/master | ||
| effect: NoSchedule | ||
| hostPID: true # Facilitate entering the host mount namespace via init | ||
| restartPolicy: Always | ||
| volumes: | ||
| - name: sentinel | ||
| hostPath: | ||
| path: /var/run | ||
| type: Directory | ||
| containers: | ||
| - name: kured | ||
| # If you find yourself here wondering why there is no | ||
| # :latest tag on Docker Hub,see the FAQ in the README | ||
| image: ghcr.io/kubereboot/kured:1.13.2 | ||
| imagePullPolicy: IfNotPresent | ||
| securityContext: | ||
| privileged: false # Give permission to nsenter /proc/1/ns/mnt | ||
| readOnlyRootFilesystem: true | ||
| allowPrivilegeEscalation: false | ||
| capabilities: | ||
| drop: ["*"] | ||
| add: ["CAP_KILL"] | ||
| ports: | ||
| - containerPort: 8080 | ||
| name: metrics | ||
| env: | ||
| # Pass in the name of the node on which this pod is scheduled | ||
| # for use with drain/uncordon operations and lock acquisition | ||
| - name: KURED_NODE_ID | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| volumeMounts: | ||
| - mountPath: /sentinel | ||
| name: sentinel | ||
| readOnly: true | ||
| command: | ||
| - /usr/bin/kured | ||
| - --reboot-sentinel=/sentinel/reboot-required | ||
| - --reboot-method=signal | ||
| # - --reboot-signal=39 | ||
| # - --force-reboot=false | ||
| # - --drain-grace-period=-1 | ||
| # - --skip-wait-for-delete-timeout=0 | ||
| # - --drain-timeout=0 | ||
| # - --period=1h | ||
| # - --ds-namespace=kube-system | ||
| # - --ds-name=kured | ||
| # - --lock-annotation=weave.works/kured-node-lock | ||
| # - --lock-ttl=0 | ||
| # - --prometheus-url=http://prometheus.monitoring.svc.cluster.local | ||
| # - --alert-filter-regexp=^RebootRequired$ | ||
| # - --alert-firing-only=false | ||
| # - --prefer-no-schedule-taint="" | ||
| # - --reboot-sentinel-command="" | ||
| # - --slack-hook-url=https://hooks.slack.com/... | ||
| # - --slack-username=prod | ||
| # - --slack-channel=alerting | ||
| # - --notify-url="" # See also shoutrrr url format | ||
| # - --message-template-drain=Draining node %s | ||
| # - --message-template-reboot=Rebooting node %s | ||
| # - --message-template-uncordon=Node %s rebooted & uncordoned successfully! | ||
| # - --blocking-pod-selector=runtime=long,cost=expensive | ||
| # - --blocking-pod-selector=name=temperamental | ||
| # - --blocking-pod-selector=... | ||
| # - --reboot-days=sun,mon,tue,wed,thu,fri,sat | ||
| # - --reboot-delay=90s | ||
| # - --start-time=0:00 | ||
| # - --end-time=23:59:59 | ||
| # - --time-zone=UTC | ||
| # - --annotate-nodes=false | ||
| # - --lock-release-delay=30m | ||
| # - --log-format=text |
Oops, something went wrong.