internal/ci: separate tip deploy workflow

Carve out the tip deploy aspects of the current trybot workflows into a
separate workflow. This allows the tip deploy and trybots to run "in
parallel" when triggered by either a new commit on the tip of this repo,
or a new commit on the tip of cue-lang/cue (the main CUE repo).

However this also allows us to limit the concurrency on the tip deploy
workflow to 1. This should prevent us hitting races that occur when
multiple commits land "at the same time" in the main CUE repo, each
triggering a tip deploy. We do not, however, configure
'cancel-in-progress' as part of the concurrency setup because of the
following GitHub bug:

    https://github.com/orgs/community/discussions/50725

Note that we leave the trybots to be triggerable via a workflow dispatch
because they need to be triggerd as part of the daily post cache
eviction, to repopulate caches. The addition of the tipdeploy workflow
will require a corresponding update in the main CUE repo to trigger the
new workflow when this CL lands.

Also turn off verbose 'go get' logging in
_scripts/tipUseAlternativeCUE.bash.

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7
Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":9,"ref":"refs/changes/60/1204960/9","targetBranch":"master"}

.github/workflows/tipdeploy.yaml

@@ -0,0 +1,185 @@
+# Code generated internal/ci/ci_tool.cue; DO NOT EDIT.
+
+name: tip deploy
+"on":
+  push:
+    branches:
+      - ci/test
+      - master
+  workflow_dispatch:
+    inputs:
+      scheduled:
+        description: Whether a workflow_dispatch was itself the result of a scheduled dispatch
+        required: true
+        default: "false"
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+    concurrency:
+      group: tip deploy
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Reset git directory modification times
+        run: touch -t 202211302355 $(find * -type d)
+      - name: Restore git file modification times
+        uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
+        run: |-
+          x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
+          if [[ "$x" == "" ]]
+          then
+             # Some steps rely on the presence or otherwise of the Dispatch-Trailer.
+             # We know that we don't have a Dispatch-Trailer in this situation,
+             # hence we use the JSON value null in order to represent that state.
+             # This means that GitHub expressions can determine whether a Dispatch-Trailer
+             # is present or not by checking whether the fromJSON() result of the
+             # output from this step is the JSON value null or not.
+             x=null
+          fi
+          echo "value<<EOD" >> $GITHUB_OUTPUT
+          echo "$x" >> $GITHUB_OUTPUT
+          echo "EOD" >> $GITHUB_OUTPUT
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
+          ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"'))
+        run: |-
+          echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
+          false
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir $HOME/.tmp
+          echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV
+        name: Set TMPDIR environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir -p ~/.lima/default
+          cat <<EOD > ~/.lima/default/lima.yaml
+          mounts:
+            - location: "~"
+          	 writable: true
+            - location: "$TMPDIR"
+          	 writable: true
+          EOD
+        name: Write lima config (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          brew install colima docker
+          colima start --mount-type virtiofs
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+        name: Install Docker (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV
+        name: Set DOCKER_HOST environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        name: Install macOS utils
+        run: brew install coreutils
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.0
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.23.0
+      - name: Set common go env vars
+        run: |-
+          go env -w GOTOOLCHAIN=local
+
+          # Dump env for good measure
+          go env
+      - if: runner.os == 'Linux'
+        name: Install Hugo (${{ runner.os }})
+        uses: peaceiris/actions-hugo@v3
+        with:
+          hugo-version: 0.128.2
+          extended: true
+      - if: runner.os == 'macOS'
+        name: Install Hugo (${{ runner.os }})
+        run: brew install hugo
+      - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true'
+        run: |
+          if ./_scripts/noWriteCache.bash HEAD
+          then
+          	echo 'Found Preprocessor-No-Write-Cache trailer'
+          	echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+          fi
+      - name: Get go mod cache directory
+        id: go-mod-cache-dir
+        run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}
+      - name: Get go build/test cache directory
+        id: go-cache-dir
+        run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        if: |-
+          (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+        uses: actions/cache@v4
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        uses: actions/cache/restore@v4
+        if: |-
+          ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+      - if: |-
+          github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test')
+        run: go clean -testcache
+      - name: Early git and code sanity checks
+        run: go run cuelang.org/go/internal/ci/[email protected]
+      - name: Perform early content checks
+        run: _scripts/contentLint.bash
+      - name: Populate CUE dependency cache
+        env:
+          CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }}
+        run: _scripts/cacheWarm.bash
+      - name: log into the central registry as porcuepine
+        run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}
+      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
+        run: _scripts/tipPatchApply.bash
+      - run: npm install
+        working-directory: hugo
+      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
+        env:
+          GOPRIVATE: cuelang.org/go
+        run: _scripts/tipUseAlternativeCUE.bash
+      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
+        run: _scripts/regenPostInfraChange.bash
+        env:
+          GOPRIVATE: cuelang.org/go
+      - name: 'tip.cuelang.org: Deploy the site'
+        run: |-
+          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
+          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
+        env:
+          GOPRIVATE: cuelang.org/go

.github/workflows/trybot.yaml

@@ -231,22 +231,3 @@ jobs:
           ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }}
           ALGOLIA_INDEX_NAME: cuelang.org
           ALGOLIA_INDEX_FILE: ../_public/algolia.json
-      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
-        run: _scripts/tipPatchApply.bash
-      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        env:
-          GOPRIVATE: cuelang.org/go
-        run: _scripts/tipUseAlternativeCUE.bash
-      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        run: _scripts/regenPostInfraChange.bash
-        env:
-          GOPRIVATE: cuelang.org/go
-      - name: 'tip.cuelang.org: Deploy the site'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        run: |-
-          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
-          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
-        env:
-          GOPRIVATE: cuelang.org/go

_scripts/tipUseAlternativeCUE.bash

@@ -25,7 +25,7 @@ td=$(mktemp -d)
 trap "rm -rf $td" EXIT
 pushd $td >/dev/null
 go mod init mod.example
-go get -v -x cuelang.org/go@$versionRef
+go get cuelang.org/go@$versionRef
 version=$(go list -m -f={{.Version}} cuelang.org/go)
 popd >/dev/null
 

internal/ci/github/tipdeploy.cue

@@ -0,0 +1,170 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"list"
+
+	"github.com/cue-tmp/jsonschema-pub/exp1/githubactions"
+)
+
+workflows: tipdeploy: _repo.bashWorkflow & {
+	name: "tip deploy"
+
+	on: {
+		push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches
+
+		// Allow the trybots to run in response to a workflow_dispatch event.
+		// One source of workflow_dispatch events is the daily_tip_check
+		// workflow.
+		workflow_dispatch: {
+			// TODO: work out how to derive this "schema" from a CUE schema
+			inputs: {
+				scheduled: {
+					description: "Whether a workflow_dispatch was itself the result of a scheduled dispatch"
+					required:    true
+					default:     "false" // this is a string type field
+				}
+			}
+		}
+	}
+
+	jobs: test: {
+		"runs-on": _repo.linuxMachine
+
+		// We only want to run this workflow in the main repo
+		if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))"
+
+		concurrency: {
+			group: "tip deploy"
+			// We do not set this to avoid getting failure messages for a cancel.
+			// "cancel-in-progress": true
+		}
+
+		steps: [
+			for v in _repo.checkoutCode {v},
+
+			for v in _installDockerMacOS {v},
+
+			_installMacOSUtils,
+			_setupBuildx,
+			_installNode,
+			for v in _installGo {v},
+			_installHugoLinux,
+			_installHugoMacOS,
+
+			// If the commit under test contains the trailer
+			// Preprocessor-No-Write-Cache: true, then set the
+			// PREPROCESSOR_NOWRITECACHE env var to non-empty.
+			githubactions.#Step & {
+				name: "Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true"
+				run: """
+					if ./_scripts/noWriteCache.bash HEAD
+					then
+						echo 'Found Preprocessor-No-Write-Cache trailer'
+						echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+					fi
+
+					"""
+			},
+
+			// cachePre must come after installing Node and Go, because the cache locations
+			// are established by running each tool.
+			for v in _setupGoActionsCaches {v},
+
+			// Run these early checks after we have restored the Go caches,
+			// as the checks are Go programs themselves.
+			_repo.earlyChecks,
+
+			githubactions.#Step & {
+				name: "Perform early content checks"
+				run:  "_scripts/contentLint.bash"
+			},
+
+			// This is the only step that needs to be given (read-only) access to the Central Registry.
+			// TODO: adopt any more specific command coming from https://cuelang.org/issue/3512.
+			// TODO: add cache dir to CI cache when it's visible via https://cuelang.org/issue/2838.
+			githubactions.#Step & {
+				name: "Populate CUE dependency cache"
+				env: CUE_TOKEN: "${{ secrets.NOTCUECKOO_CUE_TOKEN }}"
+				run: "_scripts/cacheWarm.bash"
+			},
+
+			// A number of pages that are part of cuelang.org require interacting
+			// with the Central Registry. These pages require users with slightly
+			// different access levels, in order to simulate (for example) private
+			// modules, with some users granted access whilst others are denied.
+			// The Central Registry has a special endpoint which generates access
+			// tokens for a set of such test user IDs. Access to this endpoint is
+			// sensitive, because in theory there is privilege escalation (even
+			// though in reality the test user IDs are intentionally not used
+			// for anything sensitive). As such, we use porcuepine (an account
+			// that is controlled by CUE Labs who also run the Central
+			// Registry) here in order to more carefully control in a CI
+			// environment who has access to this endpoint.
+			githubactions.#Step & {
+				name: "log into the central registry as porcuepine"
+				run:  "go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}"
+			},
+
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue"
+				run:  "_scripts/tipPatchApply.bash"
+			},
+
+			// npm install in hugo to allow serve test to pass
+			githubactions.#Step & {
+				run:                 "npm install"
+				"working-directory": "hugo"
+			},
+
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue"
+				// Only run in the main repo on the default branch or its designated test branch (i.e not CLs)
+				// so that CLs aren't blocked by failures caused by unrelated changes.
+
+				// Force Go to bypass the module proxy and sumdb for the
+				// cuelang.org/go module, ensuring that the absolute latest CUE
+				// pseudo-version is available to test against.
+				//
+				// TODO: is this really necessary? Tracking
+				// https://golang.org/issue/70042 to confirm.
+				env: GOPRIVATE: "cuelang.org/go"
+
+				run: "_scripts/tipUseAlternativeCUE.bash"
+			},
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue"
+				// Only run in the main repo on the default branch or its designated test branch (i.e not CLs)
+				// so that CLs aren't blocked by failures caused by unrelated changes.
+				run: "_scripts/regenPostInfraChange.bash"
+
+				// TODO: See comment in previous step
+				env: GOPRIVATE: "cuelang.org/go"
+			},
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Deploy the site"
+				// Only run in the main repo on the default branch or its designated test branch.
+				run: """
+				git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)"
+				_scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)'
+				"""
+
+				// TODO: See comment in previous step
+				env: GOPRIVATE: "cuelang.org/go"
+			},
+		]
+	}
+}