From 53073e58ac2196bc0b0137e781fb649cbf19532d Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Fri, 7 Jun 2024 15:10:57 +1200 Subject: [PATCH] Bring up all runners in parallel with a single Lambda call The Lambda call waits for all runners to be registered with Github before it returns. It also downloads the latest Github Actions runner version automatically and blocks automated updates so the runner doesn't go offline temporarly immediately after starting up. Hopefully this will avoid situations where the runner is running but the job hasn't been picked up. If we have reliability issues with machines not starting up properly or not accepting jobs this should make it easier to debug and fix in future, since the Lambda can log the state of the system. I've also changed the runner to mount a 32G tmpfs on /tmp, and run the runner and its working data in /tmp, which should help with some space issues. And I've moved "update the distro packages" out of the cloud-init script into `github-actions-build.sh`. It's hard to debug stuff in the cloud-init script so we want to minimize it. --- .github/workflows/build-and-test-main.yml | 21 +++++++++------------ .github/workflows/build-and-test.yml | 5 +++-- scripts/github-actions-build.sh | 2 ++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-test-main.yml b/.github/workflows/build-and-test-main.yml index 855d976dec7..e0f49048903 100644 --- a/.github/workflows/build-and-test-main.yml +++ b/.github/workflows/build-and-test-main.yml @@ -9,25 +9,22 @@ jobs: if: github.repository == 'rr-debugger/rr' steps: - id: generate - name: Generate runner ID + name: Generate runner label run: | - RUNNER_ID=rr_runner_$(uuidgen|tr -d -) - echo "RUNNER_ID=$RUNNER_ID" >> "$GITHUB_OUTPUT" - - name: Start x86-64 runner + RUNNER_LABEL=rr_runner_$(uuidgen|tr -d -) + echo "RUNNER_LABEL=$RUNNER_LABEL" >> "$GITHUB_OUTPUT" + - name: Start runners run: |2- - curl --fail -s -X POST -H "Content-Type: application/json" --data "{\"operation\": \"create\", \"architecture\":\"x86_64\", \"label\": \"${{ steps.generate.outputs.RUNNER_ID }}_x86_64\"}" https://gztdxwrnjh46z4ucjge5m4pxhu0vtfzs.lambda-url.us-east-2.on.aws - - name: Start arm64 runner - run: |2- - curl --fail -s -X POST -H "Content-Type: application/json" --data "{\"operation\": \"create\", \"architecture\":\"arm64\", \"label\": \"${{ steps.generate.outputs.RUNNER_ID }}_arm64\"}" https://gztdxwrnjh46z4ucjge5m4pxhu0vtfzs.lambda-url.us-east-2.on.aws + curl --fail -s -X POST -H "Content-Type: application/json" --data "{\"operation\": \"create\", \"architectures\":[\"x86_64\", \"arm64\"], \"label\": \"${{ steps.generate.outputs.RUNNER_LABEL }}\"}" https://bqugdqmvcql6yvz4i362qohyea0namsk.lambda-url.us-east-2.on.aws outputs: - RUNNER_ID: ${{ steps.generate.outputs.RUNNER_ID }} + RUNNER_LABEL: ${{ steps.generate.outputs.RUNNER_LABEL }} x86-64: uses: ./.github/workflows/build-and-test.yml needs: setup if: github.repository == 'rr-debugger/rr' with: - runner_id: ${{ needs.setup.outputs.RUNNER_ID }}_x86_64 + label: ${{ needs.setup.outputs.RUNNER_LABEL }} architecture: x86_64 arm64: @@ -35,7 +32,7 @@ jobs: needs: setup if: github.repository == 'rr-debugger/rr' with: - runner_id: ${{ needs.setup.outputs.RUNNER_ID }}_arm64 + label: ${{ needs.setup.outputs.RUNNER_LABEL }} architecture: arm64 stop-runners: @@ -49,4 +46,4 @@ jobs: steps: - name: "Stop runners" run: |2- - curl --fail -s -X POST -H "Content-Type: application/json" --data "{\"operation\": \"destroy\", \"labels\": [\"${{ needs.setup.outputs.RUNNER_ID }}_x86_64\", \"${{ needs.setup.outputs.RUNNER_ID }}_arm64\"]}" https://gztdxwrnjh46z4ucjge5m4pxhu0vtfzs.lambda-url.us-east-2.on.aws + curl --fail -s -X POST -H "Content-Type: application/json" --data "{\"operation\": \"destroy\", \"label\": \"${{ needs.setup.outputs.RUNNER_LABEL }}\"}" https://bqugdqmvcql6yvz4i362qohyea0namsk.lambda-url.us-east-2.on.aws diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2ad252fdf08..ba10885bd58 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -3,7 +3,7 @@ name: Build and test on runner on: workflow_call: inputs: - runner_id: + label: required: true type: string architecture: @@ -14,7 +14,8 @@ jobs: build-and-test: name: On runner runs-on: - - ${{ inputs.runner_id }} + - ${{ inputs.label }} + - arch_${{ inputs.architecture }} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/scripts/github-actions-build.sh b/scripts/github-actions-build.sh index ee6b5c070d9..54e6538e274 100755 --- a/scripts/github-actions-build.sh +++ b/scripts/github-actions-build.sh @@ -11,6 +11,8 @@ if [ $MACHINE_TYPE == 'x86_64' ]; then EXTRA_PACKAGES=g++-multilib fi +sudo apt-get update -y +sudo apt-get dist-upgrade -f -y sudo apt-get install -y $EXTRA_PACKAGES cmake g++ pkg-config zlib1g-dev git python-dev-is-python3 libacl1-dev ninja-build manpages-dev capnproto libcapnp-dev gdb lldb python3-pexpect libzstd1 libzstd-dev mkdir obj