internal/ci: separate tip deploy workflow

Carve out the tip deploy aspects of the current trybot workflows into a
separate workflow. This allows the tip deploy and trybots to run "in
parallel" when triggered by either a new commit on the tip of this repo,
or a new commit on the tip of cue-lang/cue (the main CUE repo).

However this also allows us to limit the concurrency on the tip deploy
workflow to 1. This should prevent us hitting races that occur when
multiple commits land "at the same time" in the main CUE repo, each
triggering a tip deploy. We do not, however, configure
'cancel-in-progress' as part of the concurrency setup because of the
following GitHub bug:

    https://github.com/orgs/community/discussions/50725

Note that we leave the trybots to be triggerable via a workflow dispatch
because they need to be triggerd as part of the daily post cache
eviction, to repopulate caches. The addition of the tipdeploy workflow
will require a corresponding update in the main CUE repo to trigger the
new workflow when this CL lands.

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7
Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":12,"ref":"refs/changes/60/1204960/12","targetBranch":"master"}

.github/workflows/tipdeploy.yaml

@@ -0,0 +1,182 @@
+# Code generated internal/ci/ci_tool.cue; DO NOT EDIT.
+
+name: tip deploy
+"on":
+  push:
+    branches:
+      - ci/test
+      - master
+  workflow_dispatch: {}
+concurrency:
+  group: tip deploy
+  cancel-in-progress: false
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Reset git directory modification times
+        run: touch -t 202211302355 $(find * -type d)
+      - name: Restore git file modification times
+        uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
+        run: |-
+          x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
+          if [[ "$x" == "" ]]
+          then
+             # Some steps rely on the presence or otherwise of the Dispatch-Trailer.
+             # We know that we don't have a Dispatch-Trailer in this situation,
+             # hence we use the JSON value null in order to represent that state.
+             # This means that GitHub expressions can determine whether a Dispatch-Trailer
+             # is present or not by checking whether the fromJSON() result of the
+             # output from this step is the JSON value null or not.
+             x=null
+          fi
+          echo "value<<EOD" >> $GITHUB_OUTPUT
+          echo "$x" >> $GITHUB_OUTPUT
+          echo "EOD" >> $GITHUB_OUTPUT
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
+          ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"'))
+        run: |-
+          echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
+          false
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir $HOME/.tmp
+          echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV
+        name: Set TMPDIR environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir -p ~/.lima/default
+          cat <<EOD > ~/.lima/default/lima.yaml
+          mounts:
+            - location: "~"
+          	 writable: true
+            - location: "$TMPDIR"
+          	 writable: true
+          EOD
+        name: Write lima config (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          brew install colima docker
+          colima start --mount-type virtiofs
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+        name: Install Docker (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV
+        name: Set DOCKER_HOST environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        name: Install macOS utils
+        run: brew install coreutils
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.0
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.23.0
+      - name: Set common go env vars
+        run: |-
+          go env -w GOTOOLCHAIN=local
+
+          # Dump env for good measure
+          go env
+      - if: runner.os == 'Linux'
+        name: Install Hugo (${{ runner.os }})
+        uses: peaceiris/actions-hugo@v3
+        with:
+          hugo-version: 0.128.2
+          extended: true
+      - if: runner.os == 'macOS'
+        name: Install Hugo (${{ runner.os }})
+        run: brew install hugo
+      - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true'
+        run: |
+          if ./_scripts/noWriteCache.bash HEAD
+          then
+          	echo 'Found Preprocessor-No-Write-Cache trailer'
+          	echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+          fi
+      - name: Get go mod cache directory
+        id: go-mod-cache-dir
+        run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}
+      - name: Get go build/test cache directory
+        id: go-cache-dir
+        run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        if: |-
+          (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+        uses: actions/cache@v4
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        uses: actions/cache/restore@v4
+        if: |-
+          ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+      - if: |-
+          github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test')
+        run: go clean -testcache
+      - name: Early git and code sanity checks
+        run: go run cuelang.org/go/internal/ci/[email protected]
+      - name: Perform early content checks
+        run: _scripts/contentLint.bash
+      - name: Populate CUE dependency cache
+        env:
+          CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }}
+        run: _scripts/cacheWarm.bash
+      - name: log into the central registry as porcuepine
+        run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}
+      - name: Patch the site to be compatible with the tip of cue-lang/cue
+        run: _scripts/tipPatchApply.bash
+      - name: npm install in hugo dir
+        run: npm install
+        working-directory: hugo
+      - name: Configure the site to use the tip of cue-lang/cue
+        env:
+          GOPRIVATE: cuelang.org/go
+        run: _scripts/tipUseAlternativeCUE.bash
+      - name: Build the site against the tip of cue-lang/cue
+        run: _scripts/regenPostInfraChange.bash
+        env:
+          GOPRIVATE: cuelang.org/go
+      - name: Deploy the site
+        run: |-
+          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
+          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
+        env:
+          GOPRIVATE: cuelang.org/go

.github/workflows/trybot.yaml

@@ -184,7 +184,8 @@ jobs:
         if: always()
         run: test -z "$(git status --porcelain)" || (git status; git diff; false)
       - run: ./_scripts/buildDockerImage.bash
-      - run: npm install
+      - name: npm install in hugo dir
+        run: npm install
         working-directory: hugo
       - name: Test
         run: go test ./...
@@ -231,22 +232,19 @@ jobs:
           ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }}
           ALGOLIA_INDEX_NAME: cuelang.org
           ALGOLIA_INDEX_FILE: ../_public/algolia.json
-      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
+      - name: Patch the site to be compatible with the tip of cue-lang/cue
         run: _scripts/tipPatchApply.bash
-      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        name: npm install in hugo dir
+        run: npm install
+        working-directory: hugo
+      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        name: Configure the site to use the tip of cue-lang/cue
         env:
           GOPRIVATE: cuelang.org/go
         run: _scripts/tipUseAlternativeCUE.bash
-      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+      - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        name: Build the site against the tip of cue-lang/cue
         run: _scripts/regenPostInfraChange.bash
         env:
           GOPRIVATE: cuelang.org/go
-      - name: 'tip.cuelang.org: Deploy the site'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        run: |-
-          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
-          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
-        env:
-          GOPRIVATE: cuelang.org/go

internal/ci/github/tipdeploy.cue

@@ -0,0 +1,132 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"list"
+
+	"github.com/cue-tmp/jsonschema-pub/exp1/githubactions"
+)
+
+// tipdeploy is responsible for publishing a new version of tip.cuelang.org.
+//
+// Much of this workflow follows the steps in the trybot workflow.
+workflows: tipdeploy: _repo.bashWorkflow & {
+	name: "tip deploy"
+
+	on: {
+		// Allow this workflow to run in response to a new commit at the tip of
+		// this repo. We also allow a tip deploy to be triggered from a push to
+		// ci/test to help with testing CLs.
+		push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches
+
+		// Allow this workflow to run in response to a workflow_dispatch event,
+		// specifically the event of a new commit at the tip of CUE needs to
+		// trigger a new tip deploy.
+		workflow_dispatch: {}
+	}
+
+	// We limit this workflow to run with parallelism of 1. This should
+	// prevent us hitting races that occur when multiple commits land "at the
+	// same time" in the main CUE repo, each triggering a tip deploy.
+	concurrency: {
+		group: "tip deploy"
+		// We do not set cancel-in-progress to true to avoid getting failure
+		// messages for a cancel, per the following GitHub bug:
+		//
+		//     https://github.com/orgs/community/discussions/50725
+		//
+		"cancel-in-progress": false
+	}
+
+	jobs: test: {
+		"runs-on": _repo.linuxMachine
+
+		// We only want to run this workflow in the main repo
+		if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))"
+
+		steps: [
+			for v in _repo.checkoutCode {v},
+
+			for v in _installDockerMacOS {v},
+			_installMacOSUtils,
+			_setupBuildx,
+			_installNode,
+			for v in _installGo {v},
+			_installHugoLinux,
+			_installHugoMacOS,
+
+			_setNoWriteCache,
+
+			for v in _setupGoActionsCaches {v},
+
+			_repo.earlyChecks,
+
+			_contentLint,
+
+			_cacheWarm,
+
+			_porcuepineCueLogin,
+
+			_applyTipPatches,
+
+			// npm install in hugo to allow serve test to pass which is run as
+			// part of regenPostInfraChange.bash
+			_npmInstall,
+
+			_useTipOfCUE,
+
+			_regenPostInfraChange,
+
+			{
+				name: "Deploy the site"
+				run:  """
+				git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)"
+				_scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)'
+				"""
+
+				// TODO: See comment in previous step
+				env: GOPRIVATE: "cuelang.org/go"
+			},
+		]
+	}
+}
+
+_applyTipPatches: githubactions.#Step & {
+	name: "Patch the site to be compatible with the tip of cue-lang/cue"
+	run:  "_scripts/tipPatchApply.bash"
+}
+
+_useTipOfCUE: githubactions.#Step & {
+	name: "Configure the site to use the tip of cue-lang/cue"
+
+	// Force Go to bypass the module proxy and sumdb for the
+	// cuelang.org/go module, ensuring that the absolute latest CUE
+	// pseudo-version is available to test against.
+	//
+	// TODO: is the use of GOPRIVATE (as opposed to GONOPROXY) really
+	// necessary? Tracking https://golang.org/issue/70042 to confirm.
+	env: GOPRIVATE: "cuelang.org/go"
+
+	run: "_scripts/tipUseAlternativeCUE.bash"
+}
+
+_regenPostInfraChange: githubactions.#Step & {
+	name: "Build the site against the tip of cue-lang/cue"
+	run:  "_scripts/regenPostInfraChange.bash"
+
+	// TODO: See comment in previous step
+	env: GOPRIVATE: "cuelang.org/go"
+}