COS: Add dashboard for long-term metrics #944
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Test | |
| on: | |
| pull_request: | |
| jobs: | |
| build-charm: | |
| name: Build Charm | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Remove Unnecessary Components | |
| run: | | |
| rm -rf .git | |
| rm -rf .github | |
| - name: Write lxd-profile.yaml | |
| run: | | |
| cat << EOF > ./lxd-profile.yaml | |
| config: | |
| security.nesting: true | |
| security.privileged: true | |
| raw.lxc: | | |
| lxc.apparmor.profile=unconfined | |
| lxc.mount.auto=proc:rw sys:rw cgroup:rw | |
| lxc.cgroup.devices.allow=a | |
| lxc.cap.drop= | |
| devices: | |
| kmsg: | |
| path: /dev/kmsg | |
| source: /dev/kmsg | |
| type: unix-char | |
| EOF | |
| - name: Cache github-runner Charm | |
| uses: actions/cache@v4 | |
| id: cache-charm | |
| with: | |
| path: github-runner_ubuntu-22.04-amd64.charm | |
| key: github-runner-charm-${{ hashFiles('**/*') }} | |
| - name: Setup LXD | |
| if: steps.cache-charm.outputs.cache-hit != 'true' | |
| uses: canonical/setup-lxd@main | |
| - name: Install charmcraft | |
| if: steps.cache-charm.outputs.cache-hit != 'true' | |
| run: sudo snap install charmcraft --classic | |
| - name: Pack github-runner Charm | |
| if: steps.cache-charm.outputs.cache-hit != 'true' | |
| run: charmcraft pack || ( cat ~/.local/state/charmcraft/log/* && exit 1 ) | |
| - name: Upload github-runner Charm | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm | |
| path: github-runner_ubuntu-22.04-amd64.charm | |
| run-id: | |
| name: Generate Run ID | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run-id: ${{ steps.run-id.outputs.run-id }} | |
| steps: | |
| - name: Generate Run ID | |
| id: run-id | |
| run: | | |
| echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c4)" >> $GITHUB_OUTPUT | |
| deploy-e2e-test-runner: | |
| name: Deploy End-to-End Test Runner (${{ matrix.event.name }}) | |
| runs-on: ubuntu-latest | |
| needs: [build-charm, run-id] | |
| strategy: | |
| matrix: | |
| event: | |
| - name: pull_request | |
| abbreviation: pr | |
| - name: workflow_dispatch | |
| abbreviation: wd | |
| - name: push | |
| abbreviation: push | |
| - name: schedule | |
| abbreviation: sd | |
| - name: issues | |
| abbreviation: is | |
| steps: | |
| - name: Setup Lxd Juju Controller | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| juju-channel: 3.1/stable | |
| provider: lxd | |
| - name: Install GitHub Cli | |
| run: which gh || sudo apt install gh -y | |
| - name: Create Testing Juju Model | |
| run: juju add-model testing | |
| - name: Set Testing Model Proxy Configuration | |
| run: | | |
| juju model-config juju-http-proxy=$http_proxy | |
| juju model-config juju-https-proxy=$https_proxy | |
| juju model-config juju-no-proxy=$no_proxy | |
| - name: Change Testing Model Logging Level | |
| run: juju model-config logging-config="<root>=INFO;unit=DEBUG" | |
| - name: Download github-runner Charm | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm | |
| - name: Enable br_netfilter | |
| run: sudo modprobe br_netfilter | |
| - name: Generate Runner Name | |
| id: runner-name | |
| run: echo name=${{ matrix.event.abbreviation }}-${{ needs.run-id.outputs.run-id }} >> $GITHUB_OUTPUT | |
| - name: Copy github-runner Charm | |
| run: | | |
| cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm | |
| - name: Deploy github-runner Charm (Pull Request, Workflow Dispatch and Push) | |
| if: matrix.event.name == 'workflow_dispatch' || matrix.event.name == 'push' || matrix.event.name == 'pull_request' | |
| run: | | |
| juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ | |
| ${{ steps.runner-name.outputs.name }} \ | |
| --base [email protected] \ | |
| --config path=${{ secrets.E2E_TESTING_REPO }} \ | |
| --config token=${{ secrets.E2E_TESTING_TOKEN }} \ | |
| --config virtual-machines=1 \ | |
| --config denylist=10.0.0.0/8 \ | |
| --config test-mode=insecure | |
| - name: Checkout branch (Issues, Schedule) | |
| if: matrix.event.name == 'issues' || matrix.event.name == 'schedule' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| token: ${{ secrets.E2E_TESTING_TOKEN }} | |
| - name: Create temporary orphan branch (Issues, Schedule) | |
| if: matrix.event.name == 'issues' || matrix.event.name == 'schedule' | |
| run: | | |
| # We dont need all content for the test, so create an orphan branch. | |
| git checkout --orphan ${{ steps.runner-name.outputs.name }} | |
| git reset | |
| WF_FILE=".github/workflows/schedule_issues_test.yaml" | |
| # Replace workflow event in schedule_issues_test.yaml | |
| if [[ ${{ matrix.event.name }} == 'schedule' ]]; then | |
| sed -i "s/workflow_dispatch:/schedule:\n - cron: '*\/5 * * * *'/" $WF_FILE | |
| else | |
| sed -i "s/workflow_dispatch:/issues:\n types: [opened]/" $WF_FILE | |
| fi | |
| git add $WF_FILE | |
| git config user.name github-actions | |
| git config user.email [email protected] | |
| git commit -m"Add ${{matrix.event.name}} workflow" | |
| git push origin ${{ steps.runner-name.outputs.name }} | |
| - name: Deploy github-runner Charm (Issues, Schedule) | |
| if: matrix.event.name == 'issues' || matrix.event.name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| run: | | |
| # GitHub does not allow to create multiple forks of the same repo under the same user, | |
| # so we need to create a new repository and push the branch to it. | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /user/repos \ | |
| -f name=${{ steps.runner-name.outputs.name }} | |
| TESTING_REPO=${{ secrets.E2E_TESTING_TOKEN_ORG }}/${{ steps.runner-name.outputs.name }} | |
| # Create registration token in order to allow listing of runner binaries | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| repos/${TESTING_REPO}/actions/runners/registration-token | |
| # Push the orphan branch to the newly created repo. | |
| git pull origin ${{ steps.runner-name.outputs.name }} | |
| git remote add testing https://github.com/${TESTING_REPO}.git | |
| git push testing ${{ steps.runner-name.outputs.name }}:main | |
| juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ | |
| ${{ steps.runner-name.outputs.name }} \ | |
| --base [email protected] \ | |
| --config path=$TESTING_REPO \ | |
| --config token=${{ secrets.E2E_TESTING_TOKEN }} \ | |
| --config virtual-machines=1 \ | |
| --config denylist=10.0.0.0/8 \ | |
| --config test-mode=insecure | |
| - name: Watch github-runner (Pull Request) | |
| if: matrix.event.name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| timeout-minutes: 30 | |
| run: | | |
| juju debug-log --replay --tail & | |
| while :; do | |
| JOBS=$(gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${{ secrets.E2E_TESTING_REPO }}/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs) | |
| CONCLUSION=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .conclusion') | |
| STATUS=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .status') | |
| if [[ $STATUS != "queued" && $STATUS != "in_progress" ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| if [[ $STATUS != "completed" || $CONCLUSION != "success" ]]; then | |
| echo "test workflow failed with status: $STATUS, conclusion: $CONCLUSION" | |
| kill $(jobs -p) | |
| exit 1 | |
| fi | |
| - name: Watch github-runner (Workflow Dispatch and Push) | |
| if: matrix.event.name == 'workflow_dispatch' || matrix.event.name == 'push' | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| run: | | |
| juju debug-log --replay --tail & | |
| # Base any future branches on the current branch | |
| REF_SHA=$(gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${{ secrets.E2E_TESTING_REPO }}/git/ref/heads/$GITHUB_HEAD_REF \ | |
| --jq .object.sha) | |
| # Create a temporary reference/branch | |
| # For push, this should trigger the "Push Event Tests" workflow automatically | |
| # because the test is run for branches matching the pattern "push-e2e-*" | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${{ secrets.E2E_TESTING_REPO }}/git/refs \ | |
| -f ref='refs/heads/${{ steps.runner-name.outputs.name }}' \ | |
| -f sha=$REF_SHA | |
| # For workflow_dispatch, we need to trigger the "Workflow Dispatch Tests" workflow manually | |
| if ${{ matrix.event.name == 'workflow_dispatch' }}; then | |
| gh workflow run workflow_dispatch_test.yaml \ | |
| -R ${{ secrets.E2E_TESTING_REPO }} \ | |
| --ref ${{ steps.runner-name.outputs.name }} \ | |
| -f runner=${{ steps.runner-name.outputs.name }} | |
| fi | |
| get-workflow-status() { | |
| # Search recent workflow runs for the one designated by the run-id ref | |
| output=$(gh run list \ | |
| -R ${{ secrets.E2E_TESTING_REPO }} \ | |
| -L 100 \ | |
| --json headBranch,status \ | |
| --jq '[.[] | select(.headBranch=="${{ steps.runner-name.outputs.name }}")]') | |
| # Workflows that have not started have no status | |
| if [ $(echo "$output" | jq 'length') -eq 0 ] | |
| then | |
| echo "not_started" | |
| else | |
| # Parse output with jq to get the status field of the first object | |
| status=$(echo "$output" | jq -r '.[0].status') | |
| echo "$status" | |
| fi | |
| } | |
| # Wait for the workflow to start while checking its status | |
| for i in {1..360} | |
| do | |
| status=$(get-workflow-status) | |
| echo "workflow status: $status" | |
| if [[ $status != "not_started" && $status != "queued" && $status != "in_progress" ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| # Make sure the workflow was completed or else consider it failed | |
| conclusion=$(gh run list \ | |
| -R ${{ secrets.E2E_TESTING_REPO }} \ | |
| -L 100 \ | |
| --json headBranch,conclusion \ | |
| --jq '.[] | select(.headBranch=="${{ steps.runner-name.outputs.name }}") | .conclusion') | |
| if [[ $status != "completed" || $conclusion != "success" ]]; then | |
| echo "test workflow failed with status: $status, conclusion: $conclusion" | |
| kill $(jobs -p) | |
| exit 1 | |
| else | |
| echo "Workflow completed with status: $status, conclusion: $conclusion, run-id: ${{ steps.runner-name.outputs.name }}" | |
| kill $(jobs -p) | |
| fi | |
| - name: Watch github-runner (Issues, Schedule) | |
| if: matrix.event.name == 'issues' || matrix.event.name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| run: | | |
| juju debug-log --replay --tail & | |
| TESTING_REPO=${{ secrets.E2E_TESTING_TOKEN_ORG }}/${{ steps.runner-name.outputs.name }} | |
| # For issues, we need to trigger the workflow by opening an issue | |
| if ${{ matrix.event.name == 'issues' }}; then | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${TESTING_REPO}/issues \ | |
| -f title="Test issue ${{ steps.runner-name.outputs.name }}" | |
| fi | |
| get-workflow-status() { | |
| # Search recent workflow runs for the one designated by the run-id ref | |
| output=$(gh run list \ | |
| -R ${TESTING_REPO} \ | |
| -L 100 \ | |
| --json headBranch,status,createdAt \ | |
| --jq '[.[] | select(.headBranch=="main")] | sort_by(.createdAt)') | |
| # Workflows that have not started have no status | |
| if [ $(echo "$output" | jq 'length') -eq 0 ] | |
| then | |
| echo "not_started" | |
| else | |
| # Parse output with jq to get the status field of the first object | |
| status=$(echo "$output" | jq -r '.[0].status') | |
| echo "$status" | |
| fi | |
| } | |
| # Wait for the workflow to start while checking its status | |
| for i in {1..360} | |
| do | |
| status=$(get-workflow-status) | |
| echo "workflow status: $status" | |
| if [[ $status != "not_started" && $status != "queued" && $status != "in_progress" ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| # Make sure the workflow was completed or else consider it failed | |
| runs=$(gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${TESTING_REPO}/actions/runs \ | |
| --jq '[.workflow_runs[] | select(.head_branch=="main")] | sort_by(.created_at)') | |
| conclusion=$(echo $runs | jq -r '.[0].conclusion') | |
| wf_run_id=$(echo $runs | jq -r '.[0].id') | |
| logs_filename=${{matrix.event.name}}-workflow-logs.zip | |
| # We retrieve the logs because the testing repo is deleted at the end of the test | |
| gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/${TESTING_REPO}/actions/runs/${wf_run_id}/logs > ${logs_filename} \ | |
| || (echo "Failed to retrieve logs from schedule tests" && rm ${logs_filename}) | |
| if [[ $status != "completed" || $conclusion != "success" ]]; then | |
| echo "test workflow failed with status: $status, conclusion: $conclusion" | |
| kill $(jobs -p) | |
| exit 1 | |
| else | |
| echo "Workflow completed with status: $status, conclusion: $conclusion, run-id: ${{ steps.runner-name.outputs.name }}" | |
| kill $(jobs -p) | |
| fi | |
| - name: Upload test logs (Issues, Schedule) | |
| if: always() && (matrix.event.name == 'issues' || matrix.event.name == 'schedule') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.event.name}}-workflow-logs.zip | |
| path: ${{matrix.event.name}}-workflow-logs.zip | |
| if-no-files-found: ignore | |
| - name: Show Firewall Rules | |
| run: | | |
| juju ssh ${{ steps.runner-name.outputs.name }}/0 sudo nft list ruleset | |
| - name: Clean Up (Workflow Dispatch and Push) | |
| if: always() && (matrix.event.name == 'workflow_dispatch' || matrix.event.name == 'push') | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| run: | | |
| gh api \ | |
| --method DELETE \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "/repos/${{ secrets.E2E_TESTING_REPO }}/git/refs/heads/${{ steps.runner-name.outputs.name }}" | |
| echo "Deleted ref ${{ steps.runner-name.outputs.name }}" | |
| - name: Clean Up (Issues, Schedule) | |
| if: always() && (matrix.event.name == 'issues' || matrix.event.name == 'schedule') | |
| env: | |
| GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} | |
| run: | | |
| set +e | |
| gh api \ | |
| --method DELETE \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "/repos/${{ secrets.E2E_TESTING_REPO }}/git/refs/heads/${{ steps.runner-name.outputs.name }}" \ | |
| && echo "Deleted ref ${{ steps.runner-name.outputs.name }}" | |
| TESTING_REPO=${{ secrets.E2E_TESTING_TOKEN_ORG }}/${{ steps.runner-name.outputs.name }} | |
| set -e | |
| gh api \ | |
| --method DELETE \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "/repos/${TESTING_REPO}" | |
| echo "Deleted repo ${TESTING_REPO}" | |
| e2e-test: | |
| name: End-to-End Test | |
| needs: [build-charm, run-id] | |
| runs-on: [self-hosted, linux, x64, jammy, "pr-${{ needs.run-id.outputs.run-id }}"] | |
| steps: | |
| # Snapd can have some issues in privileged LXD containers without setting | |
| # security.nesting=True and this. | |
| - name: Fix snap issue in privileged LXD containers | |
| run: ln -s /bin/true /usr/local/bin/udevadm | |
| # Below is a series of simple tests to assess the functionality of the newly spawned runner. | |
| - name: Echo hello world | |
| run: echo "hello world" | |
| - name: File permission for /usr/local/bin | |
| run: ls -ld /usr/local/bin | grep drwxrwxrwx | |
| - name: Test file permission for /usr/local/bin | |
| run: touch /usr/local/bin/test_file | |
| # "Install microk8s" step will test if the proxies settings are correct. | |
| - name: Proxy set in /etc/environment | |
| run: cat /etc/environment | |
| # "Update apt in python docker container" step will test docker default proxy settings due to | |
| # pulling the python image. | |
| - name: Proxy set in docker daemon | |
| run: | | |
| [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ | |
| || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY | |
| # "Update apt in python docker container" step will test docker client default proxy settings. | |
| - name: Proxy set in docker client | |
| run: | | |
| [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ | |
| || cat /home/ubuntu/.docker/config.json | grep httpProxy | |
| - name: Install microk8s | |
| run: sudo snap install microk8s --classic | |
| - name: Wait for microk8s | |
| timeout-minutes: 10 | |
| run: microk8s status --wait-ready | |
| - name: Deploy nginx for testing | |
| run: microk8s kubectl create deployment nginx --image=nginx | |
| - name: Wait for nginx to be ready | |
| run: microk8s kubectl rollout status deployment/nginx --timeout=30m | |
| - name: Update apt in python docker container | |
| run: docker run python:3.10-slim apt-get update | |
| - name: Docker version | |
| run: docker version | |
| - name: Check python alias for python3 | |
| run: python --version | |
| - name: pip version | |
| run: python3 -m pip --version | |
| - name: npm version | |
| run: npm --version | |
| - name: shellcheck version | |
| run: shellcheck --version | |
| - name: jq version | |
| run: jq --version | |
| - name: yq version | |
| run: yq --version | |
| - name: install check-jsonschema | |
| run: python3 -m pip install check-jsonschema | |
| # `check-jsonschema` is installed using pip. The directory `~/.local/bin` needs to be added to PATH. | |
| - name: test check-jsonschema | |
| run: check-jsonschema --version | |
| - name: Test Firewall | |
| run: | | |
| HOST_IP=$(ip route | grep default | cut -f 3 -d" ") | |
| [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] | |
| - name: Test sctp support | |
| run: sudo apt-get install lksctp-tools -yq && checksctp |