internal/ci: separate tip deploy workflow

Run the tip deploy aspects of the current trybot workflows in a separate
workflow. This allows the tip deploy and trybots to run "in parallel"
when triggered by either a new commit on the tip of this repo, or a new
commit on the tip of cue-lang/cue (the main CUE repo).

However this also allows us to limit the concurrency on the tip deploy
workflow to 1. This should prevent us hitting races that occur when
multiple commits land in quick succession in the main CUE repo, each
triggering a tip deploy. We do not, however, configure
'cancel-in-progress' as part of the concurrency setup because of the
following GitHub bug:

    https://github.com/orgs/community/discussions/50725

Note that we leave the trybots to be triggerable via a workflow dispatch
because they need to be triggered as part of the daily post cache
eviction, to repopulate caches. The addition of the tipdeploy workflow
will require a corresponding update in the main CUE repo to trigger the
new workflow when this CL lands.

Note however that we leave the step of applying the patches in the
trybot workflow. This ensures that for CLs and PRs, we can at least
verify that the tip patches apply. Whether the result actually "works"
is left to a run of the new tipdeploy workflow.

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7
Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":17,"ref":"refs/changes/60/1204960/17","targetBranch":"master"}

.github/workflows/tipdeploy.yaml

@@ -0,0 +1,182 @@
+# Code generated internal/ci/ci_tool.cue; DO NOT EDIT.
+
+name: tip deploy
+"on":
+  push:
+    branches:
+      - ci/test
+      - master
+  workflow_dispatch: {}
+concurrency:
+  group: tip deploy
+  cancel-in-progress: false
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Reset git directory modification times
+        run: touch -t 202211302355 $(find * -type d)
+      - name: Restore git file modification times
+        uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
+        run: |-
+          x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
+          if [[ "$x" == "" ]]
+          then
+             # Some steps rely on the presence or otherwise of the Dispatch-Trailer.
+             # We know that we don't have a Dispatch-Trailer in this situation,
+             # hence we use the JSON value null in order to represent that state.
+             # This means that GitHub expressions can determine whether a Dispatch-Trailer
+             # is present or not by checking whether the fromJSON() result of the
+             # output from this step is the JSON value null or not.
+             x=null
+          fi
+          echo "value<<EOD" >> $GITHUB_OUTPUT
+          echo "$x" >> $GITHUB_OUTPUT
+          echo "EOD" >> $GITHUB_OUTPUT
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
+          ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"'))
+        run: |-
+          echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
+          false
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir $HOME/.tmp
+          echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV
+        name: Set TMPDIR environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir -p ~/.lima/default
+          cat <<EOD > ~/.lima/default/lima.yaml
+          mounts:
+            - location: "~"
+          	 writable: true
+            - location: "$TMPDIR"
+          	 writable: true
+          EOD
+        name: Write lima config (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          brew install colima docker
+          colima start --mount-type virtiofs
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+        name: Install Docker (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV
+        name: Set DOCKER_HOST environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        name: Install macOS utils
+        run: brew install coreutils
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.0
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.23.0
+      - name: Set common go env vars
+        run: |-
+          go env -w GOTOOLCHAIN=local
+
+          # Dump env for good measure
+          go env
+      - if: runner.os == 'Linux'
+        name: Install Hugo (${{ runner.os }})
+        uses: peaceiris/actions-hugo@v3
+        with:
+          hugo-version: 0.128.2
+          extended: true
+      - if: runner.os == 'macOS'
+        name: Install Hugo (${{ runner.os }})
+        run: brew install hugo
+      - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true'
+        run: |
+          if ./_scripts/noWriteCache.bash HEAD
+          then
+          	echo 'Found Preprocessor-No-Write-Cache trailer'
+          	echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+          fi
+      - name: Get go mod cache directory
+        id: go-mod-cache-dir
+        run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}
+      - name: Get go build/test cache directory
+        id: go-cache-dir
+        run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        if: |-
+          (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+        uses: actions/cache@v4
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        uses: actions/cache/restore@v4
+        if: |-
+          ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+      - if: |-
+          github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test')
+        run: go clean -testcache
+      - name: Early git and code sanity checks
+        run: go run cuelang.org/go/internal/ci/[email protected]
+      - name: Perform early content checks
+        run: _scripts/contentLint.bash
+      - name: Populate CUE dependency cache
+        env:
+          CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }}
+        run: _scripts/cacheWarm.bash
+      - name: log into the central registry as porcuepine
+        run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}
+      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
+        run: _scripts/tipPatchApply.bash
+      - name: npm install in hugo dir
+        run: npm install
+        working-directory: hugo
+      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
+        env:
+          GOPRIVATE: cuelang.org/go
+        run: _scripts/tipUseAlternativeCUE.bash
+      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
+        run: _scripts/regenPostInfraChange.bash
+        env:
+          GOPRIVATE: cuelang.org/go
+      - name: 'tip.cuelang.org: Deploy the site'
+        run: |-
+          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
+          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
+        env:
+          GOPRIVATE: cuelang.org/go

.github/workflows/trybot.yaml

@@ -234,24 +234,3 @@ jobs:
           ALGOLIA_INDEX_FILE: ../_public/algolia.json
       - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
         run: _scripts/tipPatchApply.bash
-      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        name: npm install in hugo dir
-        run: npm install
-        working-directory: hugo
-      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
-        env:
-          GOPRIVATE: cuelang.org/go
-        run: _scripts/tipUseAlternativeCUE.bash
-      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
-        run: _scripts/regenPostInfraChange.bash
-        env:
-          GOPRIVATE: cuelang.org/go
-      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        name: 'tip.cuelang.org: Deploy the site'
-        run: |-
-          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
-          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
-        env:
-          GOPRIVATE: cuelang.org/go

internal/ci/github/tipdeploy.cue

@@ -0,0 +1,95 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"list"
+)
+
+// tipdeploy is responsible for publishing a new version of tip.cuelang.org.
+//
+// Much of the "setup" phase of this workflow mimics and therefore reuses steps
+// in the trybot workflow.
+workflows: tipdeploy: _repo.bashWorkflow & {
+	name: "tip deploy"
+
+	on: {
+		// Cause this workflow to run in response to a new commit at the tip of
+		// this repo. We also allow a tip deploy to be triggered from a push to
+		// ci/test to help with testing CLs.
+		push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches
+
+		// Cause this workflow to run in response to a workflow_dispatch event,
+		// specifically the event of a new commit at the tip of CUE needs to
+		// trigger a new tip deploy.
+		workflow_dispatch: {}
+	}
+
+	// We limit this workflow to run with parallelism of 1. This should
+	// prevent us hitting races that occur when multiple commits land "at the
+	// same time" in the main CUE repo, each triggering a tip deploy.
+	concurrency: {
+		group: "tip deploy"
+		// We do not set cancel-in-progress to true to avoid getting failure
+		// messages for a cancel, per the following GitHub bug:
+		//
+		//     https://github.com/orgs/community/discussions/50725
+		//
+		"cancel-in-progress": false
+	}
+
+	jobs: test: {
+		"runs-on": _repo.linuxMachine
+
+		// We only want to run this workflow in the main repo
+		if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))"
+
+		steps: [
+			for v in _repo.checkoutCode {v},
+
+			for v in _installDockerMacOS {v},
+			_installMacOSUtils,
+			_setupBuildx,
+			_installNode,
+			for v in _installGo {v},
+			_installHugoLinux,
+			_installHugoMacOS,
+
+			_setNoWriteCache,
+
+			for v in _setupGoActionsCaches {v},
+
+			_repo.earlyChecks,
+
+			_contentLint,
+
+			_cacheWarm,
+
+			_porcuepineCueLogin,
+
+			_applyTipPatches,
+
+			// npm install in hugo to allow serve test to pass which is run as
+			// part of regenPostInfraChange.bash
+			_npmInstall,
+
+			_useTipOfCUE,
+
+			_regenPostInfraChange,
+
+			_deployTipCuelangOrg,
+		]
+	}
+}

internal/ci/github/trybot.cue

@@ -226,14 +226,6 @@ workflows: trybot: _repo.bashWorkflow & {
 			// application of tip patches to fail. This doesn't guarantee later
 			// success, but it's a useful early indicator.
 			_applyTipPatches,
-
-			// npm install post applying of tip patches in case there is a
-			// relevant change in the patch affecting the result of npm install.
-			_ifMainRepoDefaultBranch & _npmInstall,
-
-			_ifMainRepoDefaultBranch & _useTipOfCUE,
-			_ifMainRepoDefaultBranch & _regenPostInfraChange,
-			_ifMainRepoDefaultBranch & _deployTipCuelangOrg,
 		]
 	}