ci: All CI runners are now on-demand (#8438) #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish CLI & Engine | |
| on: | |
| push: | |
| branches: ["main"] | |
| tags: ["v**"] | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| - ready_for_review | |
| paths: | |
| # run tests in a PR when an SDK is modified... | |
| - ./sdk | |
| # ...or when we are | |
| - ./.github/workflows/engine-and-cli-publish.yml | |
| jobs: | |
| publish: | |
| if: ${{ github.repository == 'dagger/dagger' && github.event_name == 'push' }} | |
| # Use our own Dagger runner when running in the dagger/dagger repo (including PRs) | |
| runs-on: dagger-g2-v0-13-5-16c | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.23" | |
| cache-dependency-path: ".dagger/go.sum" | |
| - name: "Publish Engine" | |
| uses: ./.github/actions/call | |
| with: | |
| function: |- | |
| engine \ | |
| publish \ | |
| --image="$DAGGER_ENGINE_IMAGE" \ | |
| --tag="${{ github.ref_name}}" --tag="${{ github.sha }}" --maybe-tag-latest \ | |
| --registry="$DAGGER_ENGINE_IMAGE_REGISTRY" \ | |
| --registry-username="$DAGGER_ENGINE_IMAGE_USERNAME" \ | |
| --registry-password=env:DAGGER_ENGINE_IMAGE_PASSWORD | |
| env: | |
| DAGGER_ENGINE_IMAGE: ${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }} | |
| DAGGER_ENGINE_IMAGE_REGISTRY: ghcr.io | |
| DAGGER_ENGINE_IMAGE_USERNAME: ${{ github.actor }} | |
| DAGGER_ENGINE_IMAGE_PASSWORD: ${{ secrets.RELEASE_DAGGER_CI_TOKEN }} | |
| - name: "Publish CLI" | |
| uses: ./.github/actions/call | |
| with: | |
| function: |- | |
| cli \ | |
| publish \ | |
| --git-dir=./.git \ | |
| --github-org-name="$GH_ORG_NAME" \ | |
| --github-token=env:GITHUB_TOKEN \ | |
| --goreleaser-key=env:GORELEASER_KEY \ | |
| --aws-access-key-id=env:AWS_ACCESS_KEY_ID \ | |
| --aws-secret-access-key=env:AWS_SECRET_ACCESS_KEY \ | |
| --aws-region=env:AWS_REGION \ | |
| --aws-bucket=env:AWS_BUCKET \ | |
| --artefacts-fqdn="$ARTEFACTS_FQDN" | |
| env: | |
| GH_ORG_NAME: ${{ vars.GH_ORG_NAME }} | |
| GITHUB_TOKEN: ${{ secrets.RELEASE_DAGGER_CI_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.RELEASE_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.RELEASE_AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.RELEASE_AWS_REGION }} | |
| AWS_BUCKET: ${{ secrets.RELEASE_AWS_BUCKET }} | |
| ARTEFACTS_FQDN: ${{ vars.RELEASE_FQDN }} | |
| GORELEASER_KEY: ${{ secrets.GORELEASER_PRO_LICENSE_KEY }} | |
| - name: "Notify" | |
| uses: ./.github/actions/notify | |
| if: github.ref_name != 'main' | |
| with: | |
| message: "🚙 Engine + 🚗 CLI: https://github.com/${{ github.repository }}/releases/tag/${{ github.ref_name }}" | |
| discord-webhook: ${{ secrets.NEW_RELEASE_DISCORD_WEBHOOK }} | |
| scan-engine: | |
| runs-on: "${{ github.repository == 'dagger/dagger' && 'dagger-g2-v0-13-5-4c' || 'ubuntu-latest' }}" | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "scan" | |
| uses: ./.github/actions/call | |
| with: | |
| function: "engine scan" | |
| test-provision-macos: | |
| name: "Test SDK Provision / macos" | |
| # We want to test the SDKs in a CLI dependency bump PR, in which case publish | |
| # has to be skipped, AND after every push to main/tags, in which case publish | |
| # must run first. This is unfortunately quite annoying to express in yaml... | |
| # https://github.com/actions/runner/issues/491#issuecomment-850884422 | |
| needs: publish | |
| if: | | |
| always() && | |
| github.repository == 'dagger/dagger' && | |
| (needs.publish.result == 'success' || needs.publish.result == 'skipped') | |
| runs-on: macos-15 | |
| steps: | |
| - name: "Set CLI Test URL" | |
| run: | | |
| if [ ${{ github.event_name }} == 'push' ]; then | |
| BASE_URL="https://${{ vars.RELEASE_FQDN }}/dagger" | |
| if [ $GITHUB_REF_NAME == 'main' ]; then | |
| # this is a push to the main branch | |
| ARCHIVE_URL="${BASE_URL}/main/${GITHUB_SHA}/dagger_${GITHUB_SHA}_darwin_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/${GITHUB_SHA}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_SHA}" | |
| else | |
| # this is a tag push | |
| ARCHIVE_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/dagger_${GITHUB_REF_NAME}_darwin_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_REF_NAME}" | |
| fi | |
| else | |
| BASE_URL="https://dl.dagger.io/dagger" | |
| # this is a pr, just default to main artifacts | |
| ARCHIVE_URL="${BASE_URL}/main/head/dagger_head_darwin_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/head/checksums.txt" | |
| RUNNER_HOST="docker-image://registry.dagger.io/engine:main" | |
| fi | |
| echo "_INTERNAL_DAGGER_TEST_CLI_URL=${ARCHIVE_URL}" >> $GITHUB_ENV | |
| echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=${CHECKSUMS_URL}" >> $GITHUB_ENV | |
| echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${RUNNER_HOST}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: "Install Docker" | |
| run: | | |
| echo "Install colima..." | |
| brew install colima | |
| echo "Install docker CLI..." | |
| brew install docker | |
| echo "Start Docker daemon via Colima..." | |
| echo "⚠️ Use mount-type 9p so that launched containers can chown: https://github.com/abiosoft/colima/issues/54#issuecomment-1250217077" | |
| colima start --mount-type 9p | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.23" | |
| - name: "Test Go SDK" | |
| run: | | |
| cd sdk/go | |
| go test -v -run TestProvision ./... | |
| - uses: yezz123/setup-uv@v4 | |
| with: | |
| uv-version: "0.2.27" | |
| - name: "Test Python SDK" | |
| run: | | |
| cd sdk/python | |
| uv run pytest -xm provision | |
| - uses: actions/setup-node@v2 | |
| with: | |
| node-version: 18 | |
| - uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: 1.1.26 | |
| - name: "Test TypeScript SDK (Node)" | |
| run: | | |
| cd sdk/typescript | |
| yarn install | |
| yarn test:node -g 'Automatic Provisioned CLI Binary' | |
| - name: "Test TypeScript SDK (Bun)" | |
| run: | | |
| cd sdk/typescript | |
| yarn install | |
| yarn test:bun -g 'Automatic Provisioned CLI Binary' | |
| - name: "ALWAYS print engine logs - especially useful on failure" | |
| if: always() | |
| run: docker logs $(docker ps -q --filter name=dagger-engine) | |
| test-provision-go-linux: | |
| name: "Test SDK Provision / go / linux" | |
| needs: publish | |
| if: | | |
| always() && | |
| github.repository == 'dagger/dagger' && | |
| (needs.publish.result == 'success' || needs.publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Set CLI Test URL" | |
| run: | | |
| if [ ${{ github.event_name }} == 'push' ]; then | |
| BASE_URL="https://${{ vars.RELEASE_FQDN }}/dagger" | |
| if [ $GITHUB_REF_NAME == 'main' ]; then | |
| # this is a push to the main branch | |
| ARCHIVE_URL="${BASE_URL}/main/${GITHUB_SHA}/dagger_${GITHUB_SHA}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/${GITHUB_SHA}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_SHA}" | |
| else | |
| # this is a tag push | |
| ARCHIVE_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/dagger_${GITHUB_REF_NAME}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_REF_NAME}" | |
| fi | |
| else | |
| BASE_URL="https://dl.dagger.io/dagger" | |
| # this is a pr, just default to main artifacts | |
| ARCHIVE_URL="${BASE_URL}/main/head/dagger_head_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/head/checksums.txt" | |
| RUNNER_HOST="docker-image://registry.dagger.io/engine:main" | |
| fi | |
| echo "_INTERNAL_DAGGER_TEST_CLI_URL=${ARCHIVE_URL}" >> $GITHUB_ENV | |
| echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=${CHECKSUMS_URL}" >> $GITHUB_ENV | |
| echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${RUNNER_HOST}" >> $GITHUB_ENV | |
| shell: bash | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.23" | |
| - name: "Test Go SDK" | |
| run: | | |
| cd sdk/go | |
| go test -v -run TestProvision ./... | |
| - name: "ALWAYS print engine logs - especially useful on failure" | |
| if: always() | |
| run: docker logs $(docker ps -q --filter name=dagger-engine) | |
| test-provision-python-linux: | |
| name: "Test SDK Provision / python / linux" | |
| needs: publish | |
| if: | | |
| always() && | |
| github.repository == 'dagger/dagger' && | |
| (needs.publish.result == 'success' || needs.publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Set CLI Test URL" | |
| run: | | |
| if [ ${{ github.event_name }} == 'push' ]; then | |
| BASE_URL="https://${{ vars.RELEASE_FQDN }}/dagger" | |
| if [ $GITHUB_REF_NAME == 'main' ]; then | |
| # this is a push to the main branch | |
| ARCHIVE_URL="${BASE_URL}/main/${GITHUB_SHA}/dagger_${GITHUB_SHA}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/${GITHUB_SHA}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_SHA}" | |
| else | |
| # this is a tag push | |
| ARCHIVE_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/dagger_${GITHUB_REF_NAME}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_REF_NAME}" | |
| fi | |
| else | |
| BASE_URL="https://dl.dagger.io/dagger" | |
| # this is a pr, just default to main artifacts | |
| ARCHIVE_URL="${BASE_URL}/main/head/dagger_head_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/head/checksums.txt" | |
| RUNNER_HOST="docker-image://registry.dagger.io/engine:main" | |
| fi | |
| echo "_INTERNAL_DAGGER_TEST_CLI_URL=${ARCHIVE_URL}" >> $GITHUB_ENV | |
| echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=${CHECKSUMS_URL}" >> $GITHUB_ENV | |
| echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${RUNNER_HOST}" >> $GITHUB_ENV | |
| shell: bash | |
| - uses: yezz123/setup-uv@v4 | |
| with: | |
| uv-version: "0.2.27" | |
| - name: "Test Python SDK" | |
| run: | | |
| cd sdk/python | |
| uv run pytest -xm provision | |
| - name: "ALWAYS print engine logs - especially useful on failure" | |
| if: always() | |
| run: docker logs $(docker ps -q --filter name=dagger-engine) | |
| test-provision-typescript-linux: | |
| name: "Test SDK Provision / TypeScript / linux" | |
| needs: publish | |
| if: | | |
| always() && | |
| github.repository == 'dagger/dagger' && | |
| (needs.publish.result == 'success' || needs.publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Set CLI Test URL" | |
| run: | | |
| if [ ${{ github.event_name }} == 'push' ]; then | |
| BASE_URL="https://${{ vars.RELEASE_FQDN }}/dagger" | |
| if [ $GITHUB_REF_NAME == 'main' ]; then | |
| # this is a push to the main branch | |
| ARCHIVE_URL="${BASE_URL}/main/${GITHUB_SHA}/dagger_${GITHUB_SHA}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/${GITHUB_SHA}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_SHA}" | |
| else | |
| # this is a tag push | |
| ARCHIVE_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/dagger_${GITHUB_REF_NAME}_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/releases/${GITHUB_REF_NAME:1}/checksums.txt" | |
| RUNNER_HOST="docker-image://${{ vars.RELEASE_DAGGER_ENGINE_IMAGE }}:${GITHUB_REF_NAME}" | |
| fi | |
| else | |
| BASE_URL="https://dl.dagger.io/dagger" | |
| # this is a pr, just default to main artifacts | |
| ARCHIVE_URL="${BASE_URL}/main/head/dagger_head_linux_amd64.tar.gz" | |
| CHECKSUMS_URL="${BASE_URL}/main/head/checksums.txt" | |
| RUNNER_HOST="docker-image://registry.dagger.io/engine:main" | |
| fi | |
| echo "_INTERNAL_DAGGER_TEST_CLI_URL=${ARCHIVE_URL}" >> $GITHUB_ENV | |
| echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=${CHECKSUMS_URL}" >> $GITHUB_ENV | |
| echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${RUNNER_HOST}" >> $GITHUB_ENV | |
| shell: bash | |
| - uses: actions/setup-node@v2 | |
| with: | |
| node-version: 18 | |
| - uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: 1.1.26 | |
| - name: "Test TypeScript SDK (Node)" | |
| run: | | |
| cd sdk/typescript | |
| yarn install | |
| yarn test:node -g 'Automatic Provisioned CLI Binary' | |
| - name: "Test TypeScript SDK (Bun)" | |
| run: | | |
| cd sdk/typescript | |
| yarn install | |
| yarn test:bun -g 'Automatic Provisioned CLI Binary' | |
| - name: "ALWAYS print engine logs - especially useful on failure" | |
| if: always() | |
| run: docker logs $(docker ps -q --filter name=dagger-engine) |