[WIP] Adding taint during image pre-pulling to prevent user pod being scheduled #2326
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is a GitHub workflow defining a set of jobs with a set of steps. | |
| # ref: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions | |
| # | |
| name: Test chart | |
| # Trigger the workflow's on all PRs and pushes so that other contributors can | |
| # run tests in their own forks. Avoid triggering these tests on changes to | |
| # documentation only changes. | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| - ".github/workflows/*" | |
| - "!.github/workflows/test-chart.yaml" | |
| push: | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| - ".github/workflows/*" | |
| - "!.github/workflows/test-chart.yaml" | |
| branches-ignore: | |
| - "dependabot/**" | |
| - "pre-commit-ci-update-config" | |
| - "update-*" | |
| - "vuln-scan-*" | |
| workflow_dispatch: | |
| jobs: | |
| lint_shell_scripts: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: pip install pre-commit | |
| - name: Run all stages including shellcheck (disabled by default) | |
| run: pre-commit run --all --hook-stage manual | |
| lint_and_validate_rendered_templates: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: pip install chartpress yamllint | |
| - name: Lint and validate | |
| run: tools/templates/lint-and-validate.py | |
| - name: Lint and validate (--strict, accept failure) | |
| run: tools/templates/lint-and-validate.py --strict | |
| continue-on-error: true | |
| lint_and_validate_templates_with_schema: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # We run this job with the latest lowest helm version we support. | |
| # | |
| # helm version 3.4.X is known to cause the following error in the named | |
| # template "jupyterhub.imagePuller.daemonset.hook.checksum": | |
| # | |
| # error calling merge: reflect: call of reflect.Value.MapKeys on ptr Value | |
| # | |
| include: | |
| - helm-version: "" # latest | |
| - helm-version: v3.5.0 # minimal required version | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| . ci/common | |
| setup_helm | |
| pip install pyyaml | |
| env: | |
| HELM_VERSION: "${{ matrix.helm-version }}" | |
| - name: Generate values.schema.json | |
| run: tools/generate-json-schema.py | |
| - name: Helm lint (values.yaml) | |
| run: helm lint ./jupyterhub | |
| - name: Helm lint (lint-and-validate-values.yaml) | |
| run: helm lint ./jupyterhub --values tools/templates/lint-and-validate-values.yaml | |
| # FIXME: We can probably emit a GitHub workflow warning if these fail | |
| # instead having them show as green without a warning or similar | |
| # | |
| # NOTE: --strict means that any warning is considered an error, and there | |
| # are several warnings that we should ignore. | |
| # | |
| - name: Helm lint --strict (values.yaml) | |
| run: helm lint --strict ./jupyterhub | |
| continue-on-error: true | |
| - name: Helm lint --strict (lint-and-validate-values.yaml) | |
| run: helm lint --strict ./jupyterhub | |
| continue-on-error: true | |
| test: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| strategy: | |
| # Keep running even if one variation of the job fail | |
| fail-fast: false | |
| matrix: | |
| # We run this job multiple times with different parameterization | |
| # specified below, these parameters have no meaning on their own and | |
| # gain meaning on how job steps use them. | |
| # | |
| # k3s-version: https://github.com/rancher/k3s/tags | |
| # k3s-channel: https://update.k3s.io/v1-release/channels | |
| # | |
| include: | |
| - k3s-channel: latest | |
| test: install | |
| - k3s-channel: stable # also test hub-slim, and prePuller.hook | |
| test: install | |
| local-chart-extra-args: >- | |
| --set hub.image.name=quay.io/jupyterhub/k8s-hub-slim | |
| --set prePuller.hook.enabled=true | |
| --set prePuller.hook.pullOnlyOnChanges=true | |
| - k3s-channel: v1.28 # also test hub.existingSecret | |
| test: install | |
| local-chart-extra-args: >- | |
| --set hub.existingSecret=test-hub-existing-secret | |
| --set proxy.secretToken=aaaa1111 | |
| --set hub.cookieSecret=bbbb2222 | |
| --set hub.config.CryptKeeper.keys[0]=cccc3333 | |
| create-k8s-test-resources: true | |
| # We run three upgrade tests where we first install an already released | |
| # Helm chart version and then upgrade to the version we are now | |
| # testing: | |
| # - latest stable version (like 1.2.3) | |
| # - latest dev version (like 1.2.3-0.dev.git.5810.hf475e7a4), | |
| # - and an old version that requires a JupyterHub DB upgrade | |
| # | |
| # It can be very useful to see the "Helm diff" step's output from the | |
| # latest dev version. | |
| # | |
| # The upgrade-from input should be a chart version, or match the version | |
| # information from | |
| # https://hub.jupyter.org/helm-chart/info.json | |
| # | |
| - k3s-channel: v1.27 | |
| test: upgrade | |
| upgrade-from: stable | |
| upgrade-from-extra-args: >- | |
| --set proxy.secretToken=aaaa1111 | |
| --set hub.cookieSecret=bbbb2222 | |
| --set hub.config.CryptKeeper.keys[0]=cccc3333 | |
| --set hub.db.type=sqlite-pvc | |
| --set singleuser.storage.type=dynamic | |
| local-chart-extra-args: >- | |
| --set hub.db.type=sqlite-pvc | |
| --set singleuser.storage.type=dynamic | |
| create-k8s-test-resources: true | |
| - k3s-channel: v1.26 | |
| test: upgrade | |
| upgrade-from: dev | |
| upgrade-from-extra-args: >- | |
| --set proxy.secretToken=aaaa1111 | |
| --set hub.db.type=sqlite-pvc | |
| --set singleuser.storage.type=dynamic | |
| local-chart-extra-args: >- | |
| --set hub.db.type=sqlite-pvc | |
| --set singleuser.storage.type=dynamic | |
| - k3s-channel: v1.25 | |
| test: upgrade | |
| # We're testing hub.db.upgrade with PostgreSQL so this version must be old | |
| # enough to require a DB upgrade | |
| upgrade-from: 2.0.0 | |
| upgrade-from-extra-args: >- | |
| --set proxy.secretToken=aaaa1111 | |
| --set hub.cookieSecret=bbbb2222 | |
| --set hub.config.CryptKeeper.keys[0]=cccc3333 | |
| --set hub.db.type=postgres | |
| --set hub.db.url=postgresql+psycopg2://postgres:postgres@postgresql:5432/jupyterhub | |
| --set singleuser.storage.type=dynamic | |
| local-chart-extra-args: >- | |
| --set hub.db.type=postgres | |
| --set hub.db.url=postgresql+psycopg2://postgres:postgres@postgresql:5432/jupyterhub | |
| --set singleuser.storage.type=dynamic | |
| --set hub.db.upgrade=true | |
| create-k8s-test-resources: true | |
| # https://artifacthub.io/packages/helm/bitnami/postgresql | |
| setup-postgresql-args: >- | |
| --version=11.6.13 | |
| --set auth.enablePostgresUser=true | |
| --set auth.postgresPassword=postgres | |
| --set auth.database=jupyterhub | |
| --set audit.logHostname=true | |
| --set audit.logConnections=true | |
| --set audit.logDisconnections=true | |
| --set audit.clientMinMessages=debug | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # chartpress requires git history to set chart version and image tags | |
| # correctly | |
| fetch-depth: 0 | |
| # Starts a k8s cluster with NetworkPolicy enforcement and installs both | |
| # kubectl and helm | |
| # | |
| # ref: https://github.com/jupyterhub/action-k3s-helm/ | |
| - uses: jupyterhub/action-k3s-helm@v4 | |
| with: | |
| k3s-channel: ${{ matrix.k3s-channel }} | |
| metrics-enabled: false | |
| traefik-enabled: false | |
| docker-enabled: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| # Install a local ACME server to fill the role of Let's Encrypt (LE). We | |
| # do this as the HTTP challenge sent out by an ACME server must be able to | |
| # reach the ACME client in our autohttps pod. | |
| - name: Install local ACME server | |
| run: | | |
| helm install pebble --repo https://hub.jupyter.org/helm-chart/ pebble --values dev-config-pebble.yaml | |
| # Build our images if needed and update values.yaml with the tags | |
| - name: Install and run chartpress | |
| run: | | |
| pip3 install -r dev-requirements.txt | |
| chartpress | |
| env: | |
| DOCKER_BUILDKIT: "1" | |
| - name: Generate values.schema.json from values.schema.yaml | |
| run: | | |
| tools/generate-json-schema.py | |
| # Validate rendered helm templates against the k8s api-server with the | |
| # dedicated lint-and-validate-values.yaml config. | |
| - name: "Helm template --validate (with lint and validate config)" | |
| run: | | |
| helm template --validate jupyterhub ./jupyterhub --values tools/templates/lint-and-validate-values.yaml ${{ matrix.helm-template-validate-extra-args }} | |
| # It is only needed at this point forward as this is when we install | |
| # jupyterhub and the autohttps pod is about to start, so for CI | |
| # performance we delayed this until now and did other things in between. | |
| - name: Await local ACME server | |
| uses: jupyterhub/action-k8s-await-workloads@v3 | |
| with: | |
| timeout: 150 | |
| max-restarts: 1 | |
| - name: "(Upgrade) Install PostgreSQL server chart" | |
| if: matrix.setup-postgresql-args | |
| run: | | |
| . ./ci/common | |
| helm install postgresql postgresql --repo=https://charts.bitnami.com/bitnami ${{ matrix.setup-postgresql-args }} | |
| await_kubectl_rollout statefulset/postgresql | |
| - name: "(Upgrade) Install ${{ matrix.upgrade-from }} chart" | |
| if: matrix.test == 'upgrade' | |
| run: | | |
| # NOTE: We change the directory so jupyterhub the chart name won't be | |
| # misunderstood as the local folder name. | |
| # | |
| # https://github.com/helm/helm/issues/9244 | |
| cd ci | |
| if [ ${{ matrix.upgrade-from }} = stable ]; then | |
| UPGRADE_FROM_VERSION=$(helm show chart --repo=https://hub.jupyter.org/helm-chart/ jupyterhub | yq e '.version' -) | |
| elif [ ${{ matrix.upgrade-from }} = dev ]; then | |
| UPGRADE_FROM_VERSION=$(helm show chart --devel --repo=https://hub.jupyter.org/helm-chart/ jupyterhub | yq e '.version' -) | |
| else | |
| UPGRADE_FROM_VERSION=${{ matrix.upgrade-from }} | |
| fi | |
| echo "UPGRADE_FROM_VERSION=$UPGRADE_FROM_VERSION" >> $GITHUB_ENV | |
| echo "" | |
| echo "Installing already released jupyterhub version $UPGRADE_FROM_VERSION" | |
| helm install jupyterhub --repo https://hub.jupyter.org/helm-chart/ jupyterhub --values ../dev-config.yaml --version=$UPGRADE_FROM_VERSION ${{ matrix.upgrade-from-extra-args }} | |
| - name: "(Upgrade) Install helm diff" | |
| if: matrix.test == 'upgrade' | |
| run: | | |
| helm plugin install https://github.com/databus23/helm-diff | |
| - name: "(Upgrade) Helm diff ${{ matrix.upgrade-from }} chart with local chart" | |
| if: matrix.test == 'upgrade' | |
| run: | | |
| LOCAL_CHART_VERSION=$(cat jupyterhub/Chart.yaml | yq e '.version' -) | |
| export STRING_REPLACER_A=$LOCAL_CHART_VERSION | |
| export STRING_REPLACER_B=$UPGRADE_FROM_VERSION | |
| echo "NOTE: Helm diff upgrade won't trigger lookup functions, so it" | |
| echo " will look like we seed new passwords all the time." | |
| echo | |
| echo "NOTE: For the helm diff only, we have replaced the new chart" | |
| echo " version with the old chart version to reduce clutter." | |
| echo | |
| echo " Old version: $UPGRADE_FROM_VERSION" | |
| echo " New version: $LOCAL_CHART_VERSION (replaced)" | |
| echo | |
| helm diff upgrade --install jupyterhub ./jupyterhub --values dev-config.yaml \ | |
| --values dev-config-local-chart-extra-config.yaml \ | |
| ${{ matrix.local-chart-extra-args }} \ | |
| --show-secrets \ | |
| --context=3 \ | |
| --post-renderer=ci/string-replacer.sh | |
| - name: "(Upgrade) Await ${{ matrix.upgrade-from }} chart" | |
| if: matrix.test == 'upgrade' | |
| uses: jupyterhub/action-k8s-await-workloads@v3 | |
| with: | |
| timeout: 150 | |
| max-restarts: 1 | |
| - name: "(Upgrade) Await ${{ matrix.upgrade-from }} chart cert acquisition" | |
| if: matrix.test == 'upgrade' | |
| run: | | |
| . ./ci/common | |
| await_autohttps_tls_cert_acquisition | |
| await_autohttps_tls_cert_save | |
| - name: Create k8s test resources | |
| if: matrix.create-k8s-test-resources | |
| run: | | |
| kubectl apply -f ci/test-hub-existing-secret.yaml | |
| - name: "Install local chart" | |
| run: | | |
| helm upgrade --install jupyterhub ./jupyterhub \ | |
| --values dev-config.yaml \ | |
| --values dev-config-local-chart-extra-config.yaml \ | |
| ${{ matrix.local-chart-extra-args }} | |
| - name: "Await local chart" | |
| uses: jupyterhub/action-k8s-await-workloads@v3 | |
| with: | |
| timeout: 150 | |
| max-restarts: 1 | |
| - name: Await local chart cert acquisition | |
| run: | | |
| . ./ci/common | |
| await_autohttps_tls_cert_acquisition | |
| - name: Run tests | |
| continue-on-error: ${{ matrix.accept-failure == true }} | |
| run: | | |
| . ./ci/common | |
| # If you have problems with the tests add '--capture=no' to show stdout | |
| pytest --verbose --maxfail=2 --color=yes ./tests | |
| # ref: https://github.com/jupyterhub/action-k8s-namespace-report | |
| - name: Kubernetes namespace report | |
| uses: jupyterhub/action-k8s-namespace-report@v1 | |
| if: always() | |
| with: | |
| important-workloads: deploy/hub deploy/proxy | |
| - name: Debug PostgreSQL logs on failure | |
| if: failure() && matrix.setup-postgresql-args | |
| run: | | |
| kubectl logs statefulset/postgresql |