From 0354c9fc04b6093d8011f75000826268fc8b13cc Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Tue, 3 Dec 2024 11:03:02 +0000 Subject: [PATCH] internal/ci: separate tip deploy workflow Run the tip deploy aspects of the current trybot workflows in a separate workflow. This allows the tip deploy and trybots to run "in parallel" when triggered by either a new commit on the tip of this repo, or a new commit on the tip of cue-lang/cue (the main CUE repo). However this also allows us to limit the concurrency on the tip deploy workflow to 1. This should prevent us hitting races that occur when multiple commits land in quick succession in the main CUE repo, each triggering a tip deploy. We do not, however, configure 'cancel-in-progress' as part of the concurrency setup because of the following GitHub bug: https://github.com/orgs/community/discussions/50725 Note that we leave the trybots to be triggerable via a workflow dispatch because they need to be triggered as part of the daily post cache eviction, to repopulate caches. The addition of the tipdeploy workflow will require a corresponding update in the main CUE repo to trigger the new workflow when this CL lands. Note however that we leave the step of applying the patches in the trybot workflow. This ensures that for CLs and PRs, we can at least verify that the tip patches apply. Whether the result actually "works" is left to a run of the new tipdeploy workflow. Signed-off-by: Paul Jolly Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7 Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":17,"ref":"refs/changes/60/1204960/17","targetBranch":"master"} --- .github/workflows/tipdeploy.yaml | 182 +++++++++++++++++++++++++++++++ .github/workflows/trybot.yaml | 21 ---- internal/ci/github/tipdeploy.cue | 95 ++++++++++++++++ internal/ci/github/trybot.cue | 8 -- 4 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/tipdeploy.yaml create mode 100644 internal/ci/github/tipdeploy.cue diff --git a/.github/workflows/tipdeploy.yaml b/.github/workflows/tipdeploy.yaml new file mode 100644 index 000000000..1963cf2cf --- /dev/null +++ b/.github/workflows/tipdeploy.yaml @@ -0,0 +1,182 @@ +# Code generated internal/ci/ci_tool.cue; DO NOT EDIT. + +name: tip deploy +"on": + push: + branches: + - ci/test + - master + workflow_dispatch: {} +concurrency: + group: tip deploy + cancel-in-progress: false +jobs: + test: + runs-on: ubuntu-22.04 + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Reset git directory modification times + run: touch -t 202211302355 $(find * -type d) + - name: Restore git file modification times + uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe + - name: Try to extract Dispatch-Trailer + id: DispatchTrailer + run: |- + x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')" + if [[ "$x" == "" ]] + then + # Some steps rely on the presence or otherwise of the Dispatch-Trailer. + # We know that we don't have a Dispatch-Trailer in this situation, + # hence we use the JSON value null in order to represent that state. + # This means that GitHub expressions can determine whether a Dispatch-Trailer + # is present or not by checking whether the fromJSON() result of the + # output from this step is the JSON value null or not. + x=null + fi + echo "value<> $GITHUB_OUTPUT + echo "$x" >> $GITHUB_OUTPUT + echo "EOD" >> $GITHUB_OUTPUT + - name: Check we don't have Dispatch-Trailer on a protected branch + if: |- + ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')) + run: |- + echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch" + false + - if: runner.os == 'macOS' + run: |- + mkdir $HOME/.tmp + echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV + name: Set TMPDIR environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + mkdir -p ~/.lima/default + cat < ~/.lima/default/lima.yaml + mounts: + - location: "~" + writable: true + - location: "$TMPDIR" + writable: true + EOD + name: Write lima config (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + brew install colima docker + colima start --mount-type virtiofs + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + name: Install Docker (${{runner.os}}) + - if: runner.os == 'macOS' + run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV + name: Set DOCKER_HOST environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + name: Install macOS utils + run: brew install coreutils + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 20.9.0 + - name: Install Go + uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.0 + - name: Set common go env vars + run: |- + go env -w GOTOOLCHAIN=local + + # Dump env for good measure + go env + - if: runner.os == 'Linux' + name: Install Hugo (${{ runner.os }}) + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: 0.128.2 + extended: true + - if: runner.os == 'macOS' + name: Install Hugo (${{ runner.os }}) + run: brew install hugo + - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true' + run: | + if ./_scripts/noWriteCache.bash HEAD + then + echo 'Found Preprocessor-No-Write-Cache trailer' + echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV + fi + - name: Get go mod cache directory + id: go-mod-cache-dir + run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT} + - name: Get go build/test cache directory + id: go-cache-dir + run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT} + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + if: |- + (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + uses: actions/cache@v4 + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + uses: actions/cache/restore@v4 + if: |- + ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + - if: |- + github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test') + run: go clean -testcache + - name: Early git and code sanity checks + run: go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650 + - name: Perform early content checks + run: _scripts/contentLint.bash + - name: Populate CUE dependency cache + env: + CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }} + run: _scripts/cacheWarm.bash + - name: log into the central registry as porcuepine + run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }} + - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' + run: _scripts/tipPatchApply.bash + - name: npm install in hugo dir + run: npm install + working-directory: hugo + - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' + env: + GOPRIVATE: cuelang.org/go + run: _scripts/tipUseAlternativeCUE.bash + - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' + run: _scripts/regenPostInfraChange.bash + env: + GOPRIVATE: cuelang.org/go + - name: 'tip.cuelang.org: Deploy the site' + run: |- + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" + _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' + env: + GOPRIVATE: cuelang.org/go diff --git a/.github/workflows/trybot.yaml b/.github/workflows/trybot.yaml index b852195c6..c20c639fb 100644 --- a/.github/workflows/trybot.yaml +++ b/.github/workflows/trybot.yaml @@ -234,24 +234,3 @@ jobs: ALGOLIA_INDEX_FILE: ../_public/algolia.json - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' run: _scripts/tipPatchApply.bash - - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - name: npm install in hugo dir - run: npm install - working-directory: hugo - - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' - env: - GOPRIVATE: cuelang.org/go - run: _scripts/tipUseAlternativeCUE.bash - - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' - run: _scripts/regenPostInfraChange.bash - env: - GOPRIVATE: cuelang.org/go - - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - name: 'tip.cuelang.org: Deploy the site' - run: |- - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' - env: - GOPRIVATE: cuelang.org/go diff --git a/internal/ci/github/tipdeploy.cue b/internal/ci/github/tipdeploy.cue new file mode 100644 index 000000000..d5aace394 --- /dev/null +++ b/internal/ci/github/tipdeploy.cue @@ -0,0 +1,95 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "list" +) + +// tipdeploy is responsible for publishing a new version of tip.cuelang.org. +// +// Much of the "setup" phase of this workflow mimics and therefore reuses steps +// in the trybot workflow. +workflows: tipdeploy: _repo.bashWorkflow & { + name: "tip deploy" + + on: { + // Cause this workflow to run in response to a new commit at the tip of + // this repo. We also allow a tip deploy to be triggered from a push to + // ci/test to help with testing CLs. + push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches + + // Cause this workflow to run in response to a workflow_dispatch event, + // specifically the event of a new commit at the tip of CUE needs to + // trigger a new tip deploy. + workflow_dispatch: {} + } + + // We limit this workflow to run with parallelism of 1. This should + // prevent us hitting races that occur when multiple commits land "at the + // same time" in the main CUE repo, each triggering a tip deploy. + concurrency: { + group: "tip deploy" + // We do not set cancel-in-progress to true to avoid getting failure + // messages for a cancel, per the following GitHub bug: + // + // https://github.com/orgs/community/discussions/50725 + // + "cancel-in-progress": false + } + + jobs: test: { + "runs-on": _repo.linuxMachine + + // We only want to run this workflow in the main repo + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" + + steps: [ + for v in _repo.checkoutCode {v}, + + for v in _installDockerMacOS {v}, + _installMacOSUtils, + _setupBuildx, + _installNode, + for v in _installGo {v}, + _installHugoLinux, + _installHugoMacOS, + + _setNoWriteCache, + + for v in _setupGoActionsCaches {v}, + + _repo.earlyChecks, + + _contentLint, + + _cacheWarm, + + _porcuepineCueLogin, + + _applyTipPatches, + + // npm install in hugo to allow serve test to pass which is run as + // part of regenPostInfraChange.bash + _npmInstall, + + _useTipOfCUE, + + _regenPostInfraChange, + + _deployTipCuelangOrg, + ] + } +} diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index d8caf06b2..046f13d63 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -226,14 +226,6 @@ workflows: trybot: _repo.bashWorkflow & { // application of tip patches to fail. This doesn't guarantee later // success, but it's a useful early indicator. _applyTipPatches, - - // npm install post applying of tip patches in case there is a - // relevant change in the patch affecting the result of npm install. - _ifMainRepoDefaultBranch & _npmInstall, - - _ifMainRepoDefaultBranch & _useTipOfCUE, - _ifMainRepoDefaultBranch & _regenPostInfraChange, - _ifMainRepoDefaultBranch & _deployTipCuelangOrg, ] }