internal/ci: separate tip deploy workflow

DO NOT SUBMIT

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7
Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":5,"ref":"refs/changes/60/1204960/5","targetBranch":"master"}

.github/workflows/tipdeploy.yaml

@@ -0,0 +1,192 @@
+# Code generated internal/ci/ci_tool.cue; DO NOT EDIT.
+
+name: tip deploy
+"on":
+  push:
+    branches:
+      - ci/test
+      - master
+  workflow_dispatch:
+    inputs:
+      scheduled:
+        description: Whether a workflow_dispatch was itself the result of a scheduled dispatch
+        required: true
+        default: "false"
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    concurrency:
+      group: tip deploy
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Reset git directory modification times
+        run: touch -t 202211302355 $(find * -type d)
+      - name: Restore git file modification times
+        uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
+        run: |-
+          x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
+          if [[ "$x" == "" ]]
+          then
+             # Some steps rely on the presence or otherwise of the Dispatch-Trailer.
+             # We know that we don't have a Dispatch-Trailer in this situation,
+             # hence we use the JSON value null in order to represent that state.
+             # This means that GitHub expressions can determine whether a Dispatch-Trailer
+             # is present or not by checking whether the fromJSON() result of the
+             # output from this step is the JSON value null or not.
+             x=null
+          fi
+          echo "value<<EOD" >> $GITHUB_OUTPUT
+          echo "$x" >> $GITHUB_OUTPUT
+          echo "EOD" >> $GITHUB_OUTPUT
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
+          ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"'))
+        run: |-
+          echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
+          false
+      - name: Fail if Preprocessor-No-Write-Cache trailer is present for a scheduled workflow run
+        if: github.event.inputs.scheduled == 'true'
+        run: '! ./_scripts/noWriteCache.bash HEAD'
+      - if: github.repository != 'cue-lang/cuelang.org'
+        run: |
+          echo 'Setting CI_NO_SKIP_CACHE=true'
+          echo "CI_NO_SKIP_CACHE=true" >> $GITHUB_ENV
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir $HOME/.tmp
+          echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV
+        name: Set TMPDIR environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          mkdir -p ~/.lima/default
+          cat <<EOD > ~/.lima/default/lima.yaml
+          mounts:
+            - location: "~"
+          	 writable: true
+            - location: "$TMPDIR"
+          	 writable: true
+          EOD
+        name: Write lima config (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: |-
+          brew install colima docker
+          colima start --mount-type virtiofs
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+        name: Install Docker (${{runner.os}})
+      - if: runner.os == 'macOS'
+        run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV
+        name: Set DOCKER_HOST environment variable (${{runner.os}})
+      - if: runner.os == 'macOS'
+        name: Install macOS utils
+        run: brew install coreutils
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.0
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.23.0
+      - name: Set common go env vars
+        run: |-
+          go env -w GOTOOLCHAIN=local
+
+          # Dump env for good measure
+          go env
+      - if: runner.os == 'Linux'
+        name: Install Hugo (${{ runner.os }})
+        uses: peaceiris/actions-hugo@v3
+        with:
+          hugo-version: 0.128.2
+          extended: true
+      - if: runner.os == 'macOS'
+        name: Install Hugo (${{ runner.os }})
+        run: brew install hugo
+      - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true'
+        run: |
+          if ./_scripts/noWriteCache.bash HEAD
+          then
+          	echo 'Found Preprocessor-No-Write-Cache trailer'
+          	echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+          fi
+      - name: Get go mod cache directory
+        id: go-mod-cache-dir
+        run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}
+      - name: Get go build/test cache directory
+        id: go-cache-dir
+        run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        if: |-
+          (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+        uses: actions/cache@v4
+      - with:
+          path: |-
+            ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
+            ${{ steps.go-cache-dir.outputs.dir }}
+            ~/.cache/dockercache
+            ~/.cache/node-gyp
+            ~/.npm
+            ${{ github.workspace }}/playground/.webpack_cache
+          key: ${{ runner.os }}-1.23.0-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-1.23.0
+        uses: actions/cache/restore@v4
+        if: |-
+          ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
+      - if: |-
+          github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test')
+        run: go clean -testcache
+      - name: Early git and code sanity checks
+        run: go run cuelang.org/go/internal/ci/[email protected]
+      - name: Perform early content checks
+        run: _scripts/contentLint.bash
+      - name: Populate CUE dependency cache
+        env:
+          CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }}
+        run: _scripts/cacheWarm.bash
+      - name: log into the central registry as porcuepine
+        run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}
+      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
+        run: _scripts/tipPatchApply.bash
+      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
+        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        env:
+          GOPRIVATE: cuelang.org/go
+        run: _scripts/tipUseAlternativeCUE.bash
+      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
+        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        run: _scripts/regenPostInfraChange.bash
+        env:
+          GOPRIVATE: cuelang.org/go
+      - name: 'tip.cuelang.org: Deploy the site'
+        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
+        run: |-
+          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
+          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
+        env:
+          GOPRIVATE: cuelang.org/go

.github/workflows/trybot.yaml

@@ -231,22 +231,3 @@ jobs:
           ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }}
           ALGOLIA_INDEX_NAME: cuelang.org
           ALGOLIA_INDEX_FILE: ../_public/algolia.json
-      - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue'
-        run: _scripts/tipPatchApply.bash
-      - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        env:
-          GOPRIVATE: cuelang.org/go
-        run: _scripts/tipUseAlternativeCUE.bash
-      - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        run: _scripts/regenPostInfraChange.bash
-        env:
-          GOPRIVATE: cuelang.org/go
-      - name: 'tip.cuelang.org: Deploy the site'
-        if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test'))
-        run: |-
-          git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
-          _scripts/tipDeploy.bash 'cueckoo' '[email protected]'
-        env:
-          GOPRIVATE: cuelang.org/go

internal/ci/github/tipdeploy.cue

@@ -0,0 +1,174 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"list"
+	"strings"
+	"strconv"
+
+	"github.com/cue-tmp/jsonschema-pub/exp1/githubactions"
+
+	"github.com/cue-lang/cuelang.org/internal/ci/netlify"
+)
+
+workflows: tipdeploy: _repo.bashWorkflow & {
+	name: "tip deploy"
+
+	on: {
+		push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches
+
+		// Allow the trybots to run in response to a workflow_dispatch event.
+		// One source of workflow_dispatch events is the daily_tip_check
+		// workflow.
+		workflow_dispatch: {
+			// TODO: work out how to derive this "schema" from a CUE schema
+			inputs: {
+				scheduled: {
+					description: "Whether a workflow_dispatch was itself the result of a scheduled dispatch"
+					required:    true
+					default:     "false" // this is a string type field
+				}
+			}
+		}
+	}
+
+	jobs: test: {
+		"runs-on": _repo.linuxMachine
+
+		// We only want to run this workflow in the main repo
+		if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))"
+
+		concurrency: {
+			group: "tip deploy"
+			// We do not set this to avoid getting failure messages for a cancel.
+			// "cancel-in-progress": true
+		}
+
+		steps: [
+			for v in _repo.checkoutCode {v},
+
+			for v in _installDockerMacOS {v},
+
+			_installMacOSUtils,
+			_setupBuildx,
+			_installNode,
+			for v in _installGo {v},
+			_installHugoLinux,
+			_installHugoMacOS,
+
+			// If the commit under test contains the trailer
+			// Preprocessor-No-Write-Cache: true, then set the
+			// PREPROCESSOR_NOWRITECACHE env var to non-empty.
+			githubactions.#Step & {
+				name: "Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true"
+				run: """
+					if ./_scripts/noWriteCache.bash HEAD
+					then
+						echo 'Found Preprocessor-No-Write-Cache trailer'
+						echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV
+					fi
+
+					"""
+			},
+
+			// cachePre must come after installing Node and Go, because the cache locations
+			// are established by running each tool.
+			for v in _setupGoActionsCaches {v},
+
+			// Run these early checks after we have restored the Go caches,
+			// as the checks are Go programs themselves.
+			_repo.earlyChecks,
+
+			githubactions.#Step & {
+				name: "Perform early content checks"
+				run:  "_scripts/contentLint.bash"
+			},
+
+			// This is the only step that needs to be given (read-only) access to the Central Registry.
+			// TODO: adopt any more specific command coming from https://cuelang.org/issue/3512.
+			// TODO: add cache dir to CI cache when it's visible via https://cuelang.org/issue/2838.
+			githubactions.#Step & {
+				name: "Populate CUE dependency cache"
+				env: CUE_TOKEN: "${{ secrets.NOTCUECKOO_CUE_TOKEN }}"
+				run: "_scripts/cacheWarm.bash"
+			},
+
+			// A number of pages that are part of cuelang.org require interacting
+			// with the Central Registry. These pages require users with slightly
+			// different access levels, in order to simulate (for example) private
+			// modules, with some users granted access whilst others are denied.
+			// The Central Registry has a special endpoint which generates access
+			// tokens for a set of such test user IDs. Access to this endpoint is
+			// sensitive, because in theory there is privilege escalation (even
+			// though in reality the test user IDs are intentionally not used
+			// for anything sensitive). As such, we use porcuepine (an account
+			// that is controlled by CUE Labs who also run the Central
+			// Registry) here in order to more carefully control in a CI
+			// environment who has access to this endpoint.
+			githubactions.#Step & {
+				name: "log into the central registry as porcuepine"
+				run:  "go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}"
+			},
+
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue"
+				run:  "_scripts/tipPatchApply.bash"
+			},
+
+			// npm install in hugo to allow serve test to pass
+			githubactions.#Step & {
+				run:                 "npm install"
+				"working-directory": "hugo"
+			},
+
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue"
+				// Only run in the main repo on the default branch or its designated test branch (i.e not CLs)
+				// so that CLs aren't blocked by failures caused by unrelated changes.
+
+				// Force Go to bypass the module proxy and sumdb for the
+				// cuelang.org/go module, ensuring that the absolute latest CUE
+				// pseudo-version is available to test against.
+				//
+				// TODO: is this really necessary? Tracking
+				// https://golang.org/issue/70042 to confirm.
+				env: GOPRIVATE: "cuelang.org/go"
+
+				run: "_scripts/tipUseAlternativeCUE.bash"
+			},
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue"
+				// Only run in the main repo on the default branch or its designated test branch (i.e not CLs)
+				// so that CLs aren't blocked by failures caused by unrelated changes.
+				run: "_scripts/regenPostInfraChange.bash"
+
+				// TODO: See comment in previous step
+				env: GOPRIVATE: "cuelang.org/go"
+			},
+			githubactions.#Step & {
+				name: "tip.cuelang.org: Deploy the site"
+				// Only run in the main repo on the default branch or its designated test branch.
+				run: """
+				git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)"
+				_scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)'
+				"""
+
+				// TODO: See comment in previous step
+				env: GOPRIVATE: "cuelang.org/go"
+			},
+		]
+	}
+}