From 462a2d2d4e1ec5286ff944f656ec653543d9b000 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Sun, 1 Dec 2024 08:48:26 +0000 Subject: [PATCH] internal/ci: separate tip deploy workflow Carve out the tip deploy aspects of the current trybot workflows into a separate workflow. This allows the tip deploy and trybots to run "in parallel" when triggered by either a new commit on the tip of this repo, or a new commit on the tip of cue-lang/cue (the main CUE repo). However this also allows us to limit the concurrency on the tip deploy workflow to 1. This should prevent us hitting races that occur when multiple commits land "at the same time" in the main CUE repo, each triggering a tip deploy. We do not, however, configure 'cancel-in-progress' as part of the concurrency setup because of the following GitHub bug: https://github.com/orgs/community/discussions/50725 Note that we leave the trybots to be triggerable via a workflow dispatch because they need to be triggerd as part of the daily post cache eviction, to repopulate caches. The addition of the tipdeploy workflow will require a corresponding update in the main CUE repo to trigger the new workflow when this CL lands. Signed-off-by: Paul Jolly Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7 Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":12,"ref":"refs/changes/60/1204960/12","targetBranch":"master"} --- .github/workflows/tipdeploy.yaml | 182 +++++++++++++++++++++++++++++++ .github/workflows/trybot.yaml | 24 ++-- internal/ci/github/tipdeploy.cue | 132 ++++++++++++++++++++++ internal/ci/github/trybot.cue | 55 +++------- 4 files changed, 339 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/tipdeploy.yaml create mode 100644 internal/ci/github/tipdeploy.cue diff --git a/.github/workflows/tipdeploy.yaml b/.github/workflows/tipdeploy.yaml new file mode 100644 index 0000000000..d073cd0f26 --- /dev/null +++ b/.github/workflows/tipdeploy.yaml @@ -0,0 +1,182 @@ +# Code generated internal/ci/ci_tool.cue; DO NOT EDIT. + +name: tip deploy +"on": + push: + branches: + - ci/test + - master + workflow_dispatch: {} +concurrency: + group: tip deploy + cancel-in-progress: false +jobs: + test: + runs-on: ubuntu-22.04 + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Reset git directory modification times + run: touch -t 202211302355 $(find * -type d) + - name: Restore git file modification times + uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe + - name: Try to extract Dispatch-Trailer + id: DispatchTrailer + run: |- + x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')" + if [[ "$x" == "" ]] + then + # Some steps rely on the presence or otherwise of the Dispatch-Trailer. + # We know that we don't have a Dispatch-Trailer in this situation, + # hence we use the JSON value null in order to represent that state. + # This means that GitHub expressions can determine whether a Dispatch-Trailer + # is present or not by checking whether the fromJSON() result of the + # output from this step is the JSON value null or not. + x=null + fi + echo "value<> $GITHUB_OUTPUT + echo "$x" >> $GITHUB_OUTPUT + echo "EOD" >> $GITHUB_OUTPUT + - name: Check we don't have Dispatch-Trailer on a protected branch + if: |- + ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')) + run: |- + echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch" + false + - if: runner.os == 'macOS' + run: |- + mkdir $HOME/.tmp + echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV + name: Set TMPDIR environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + mkdir -p ~/.lima/default + cat < ~/.lima/default/lima.yaml + mounts: + - location: "~" + writable: true + - location: "$TMPDIR" + writable: true + EOD + name: Write lima config (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + brew install colima docker + colima start --mount-type virtiofs + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + name: Install Docker (${{runner.os}}) + - if: runner.os == 'macOS' + run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV + name: Set DOCKER_HOST environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + name: Install macOS utils + run: brew install coreutils + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 20.9.0 + - name: Install Go + uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.0 + - name: Set common go env vars + run: |- + go env -w GOTOOLCHAIN=local + + # Dump env for good measure + go env + - if: runner.os == 'Linux' + name: Install Hugo (${{ runner.os }}) + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: 0.128.2 + extended: true + - if: runner.os == 'macOS' + name: Install Hugo (${{ runner.os }}) + run: brew install hugo + - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true' + run: | + if ./_scripts/noWriteCache.bash HEAD + then + echo 'Found Preprocessor-No-Write-Cache trailer' + echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV + fi + - name: Get go mod cache directory + id: go-mod-cache-dir + run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT} + - name: Get go build/test cache directory + id: go-cache-dir + run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT} + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + if: |- + (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + uses: actions/cache@v4 + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + uses: actions/cache/restore@v4 + if: |- + ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + - if: |- + github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test') + run: go clean -testcache + - name: Early git and code sanity checks + run: go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650 + - name: Perform early content checks + run: _scripts/contentLint.bash + - name: Populate CUE dependency cache + env: + CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }} + run: _scripts/cacheWarm.bash + - name: log into the central registry as porcuepine + run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }} + - name: Patch the site to be compatible with the tip of cue-lang/cue + run: _scripts/tipPatchApply.bash + - name: npm install in hugo dir + run: npm install + working-directory: hugo + - name: Configure the site to use the tip of cue-lang/cue + env: + GOPRIVATE: cuelang.org/go + run: _scripts/tipUseAlternativeCUE.bash + - name: Build the site against the tip of cue-lang/cue + run: _scripts/regenPostInfraChange.bash + env: + GOPRIVATE: cuelang.org/go + - name: Deploy the site + run: |- + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" + _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' + env: + GOPRIVATE: cuelang.org/go diff --git a/.github/workflows/trybot.yaml b/.github/workflows/trybot.yaml index 0bd61905c9..9dba27404f 100644 --- a/.github/workflows/trybot.yaml +++ b/.github/workflows/trybot.yaml @@ -184,7 +184,8 @@ jobs: if: always() run: test -z "$(git status --porcelain)" || (git status; git diff; false) - run: ./_scripts/buildDockerImage.bash - - run: npm install + - name: npm install in hugo dir + run: npm install working-directory: hugo - name: Test run: go test ./... @@ -231,22 +232,19 @@ jobs: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }} ALGOLIA_INDEX_NAME: cuelang.org ALGOLIA_INDEX_FILE: ../_public/algolia.json - - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' + - name: Patch the site to be compatible with the tip of cue-lang/cue run: _scripts/tipPatchApply.bash - - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: npm install in hugo dir + run: npm install + working-directory: hugo + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: Configure the site to use the tip of cue-lang/cue env: GOPRIVATE: cuelang.org/go run: _scripts/tipUseAlternativeCUE.bash - - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: Build the site against the tip of cue-lang/cue run: _scripts/regenPostInfraChange.bash env: GOPRIVATE: cuelang.org/go - - name: 'tip.cuelang.org: Deploy the site' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - run: |- - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' - env: - GOPRIVATE: cuelang.org/go diff --git a/internal/ci/github/tipdeploy.cue b/internal/ci/github/tipdeploy.cue new file mode 100644 index 0000000000..d51d92bab8 --- /dev/null +++ b/internal/ci/github/tipdeploy.cue @@ -0,0 +1,132 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "list" + + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" +) + +// tipdeploy is responsible for publishing a new version of tip.cuelang.org. +// +// Much of this workflow follows the steps in the trybot workflow. +workflows: tipdeploy: _repo.bashWorkflow & { + name: "tip deploy" + + on: { + // Allow this workflow to run in response to a new commit at the tip of + // this repo. We also allow a tip deploy to be triggered from a push to + // ci/test to help with testing CLs. + push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches + + // Allow this workflow to run in response to a workflow_dispatch event, + // specifically the event of a new commit at the tip of CUE needs to + // trigger a new tip deploy. + workflow_dispatch: {} + } + + // We limit this workflow to run with parallelism of 1. This should + // prevent us hitting races that occur when multiple commits land "at the + // same time" in the main CUE repo, each triggering a tip deploy. + concurrency: { + group: "tip deploy" + // We do not set cancel-in-progress to true to avoid getting failure + // messages for a cancel, per the following GitHub bug: + // + // https://github.com/orgs/community/discussions/50725 + // + "cancel-in-progress": false + } + + jobs: test: { + "runs-on": _repo.linuxMachine + + // We only want to run this workflow in the main repo + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" + + steps: [ + for v in _repo.checkoutCode {v}, + + for v in _installDockerMacOS {v}, + _installMacOSUtils, + _setupBuildx, + _installNode, + for v in _installGo {v}, + _installHugoLinux, + _installHugoMacOS, + + _setNoWriteCache, + + for v in _setupGoActionsCaches {v}, + + _repo.earlyChecks, + + _contentLint, + + _cacheWarm, + + _porcuepineCueLogin, + + _applyTipPatches, + + // npm install in hugo to allow serve test to pass which is run as + // part of regenPostInfraChange.bash + _npmInstall, + + _useTipOfCUE, + + _regenPostInfraChange, + + { + name: "Deploy the site" + run: """ + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" + _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' + """ + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" + }, + ] + } +} + +_applyTipPatches: githubactions.#Step & { + name: "Patch the site to be compatible with the tip of cue-lang/cue" + run: "_scripts/tipPatchApply.bash" +} + +_useTipOfCUE: githubactions.#Step & { + name: "Configure the site to use the tip of cue-lang/cue" + + // Force Go to bypass the module proxy and sumdb for the + // cuelang.org/go module, ensuring that the absolute latest CUE + // pseudo-version is available to test against. + // + // TODO: is the use of GOPRIVATE (as opposed to GONOPROXY) really + // necessary? Tracking https://golang.org/issue/70042 to confirm. + env: GOPRIVATE: "cuelang.org/go" + + run: "_scripts/tipUseAlternativeCUE.bash" +} + +_regenPostInfraChange: githubactions.#Step & { + name: "Build the site against the tip of cue-lang/cue" + run: "_scripts/regenPostInfraChange.bash" + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" +} diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 2422ddb586..8bc8fb0306 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -222,49 +222,17 @@ workflows: trybot: _repo.bashWorkflow & { } }, - githubactions.#Step & { - name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue" - run: "_scripts/tipPatchApply.bash" - }, + // We do this on all branches to catch changes in CL that cause the + // application of tip patches to fail. This doesn't guarantee later + // success, but it's a useful early indicator. + _applyTipPatches, - githubactions.#Step & { - name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - - // Force Go to bypass the module proxy and sumdb for the - // cuelang.org/go module, ensuring that the absolute latest CUE - // pseudo-version is available to test against. - // - // TODO: is this really necessary? Tracking - // https://golang.org/issue/70042 to confirm. - env: GOPRIVATE: "cuelang.org/go" - - run: "_scripts/tipUseAlternativeCUE.bash" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: "_scripts/regenPostInfraChange.bash" - - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Deploy the site" - // Only run in the main repo on the default branch or its designated test branch. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: """ - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" - _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' - """ + // npm install post applying of tip patches in case there is an relevant patch + _mainRepoDefaulBranch & _npmInstall, - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, + _mainRepoDefaulBranch & _useTipOfCUE, + + _mainRepoDefaulBranch & _regenPostInfraChange, ] } @@ -286,6 +254,10 @@ workflows: trybot: _repo.bashWorkflow & { } } +_mainRepoDefaulBranch: githubactions.#Step & { + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" +} + _installNode: githubactions.#Step & { name: "Install Node" uses: "actions/setup-node@v4" @@ -488,6 +460,7 @@ _porcuepineCueLogin: githubactions.#Step & { // npm install in hugo to allow serve test to pass _npmInstall: githubactions.#Step & { + name: "npm install in hugo dir" run: "npm install" "working-directory": "hugo" }