From bf760e3408a2eb4b226697135c5db4dc3e6a238d Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Fri, 29 Nov 2024 15:21:23 +0000 Subject: [PATCH] internal/ci: separate tip deploy workflow DO NOT SUBMIT Signed-off-by: Paul Jolly Change-Id: Icee0d3ec98b04e2ad02209fbb23f85f7edaa8ee7 Dispatch-Trailer: {"type":"trybot","CL":1204960,"patchset":7,"ref":"refs/changes/60/1204960/7","targetBranch":"master"} --- .github/workflows/tipdeploy.yaml | 192 +++++++++++++++++++++++++++++++ .github/workflows/trybot.yaml | 19 --- internal/ci/github/tipdeploy.cue | 174 ++++++++++++++++++++++++++++ internal/ci/github/trybot.cue | 46 +------- 4 files changed, 367 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/tipdeploy.yaml create mode 100644 internal/ci/github/tipdeploy.cue diff --git a/.github/workflows/tipdeploy.yaml b/.github/workflows/tipdeploy.yaml new file mode 100644 index 000000000..a3475290a --- /dev/null +++ b/.github/workflows/tipdeploy.yaml @@ -0,0 +1,192 @@ +# Code generated internal/ci/ci_tool.cue; DO NOT EDIT. + +name: tip deploy +"on": + push: + branches: + - ci/test + - master + workflow_dispatch: + inputs: + scheduled: + description: Whether a workflow_dispatch was itself the result of a scheduled dispatch + required: true + default: "false" +jobs: + test: + runs-on: ubuntu-22.04 + concurrency: + group: tip deploy + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Reset git directory modification times + run: touch -t 202211302355 $(find * -type d) + - name: Restore git file modification times + uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe + - name: Try to extract Dispatch-Trailer + id: DispatchTrailer + run: |- + x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')" + if [[ "$x" == "" ]] + then + # Some steps rely on the presence or otherwise of the Dispatch-Trailer. + # We know that we don't have a Dispatch-Trailer in this situation, + # hence we use the JSON value null in order to represent that state. + # This means that GitHub expressions can determine whether a Dispatch-Trailer + # is present or not by checking whether the fromJSON() result of the + # output from this step is the JSON value null or not. + x=null + fi + echo "value<> $GITHUB_OUTPUT + echo "$x" >> $GITHUB_OUTPUT + echo "EOD" >> $GITHUB_OUTPUT + - name: Check we don't have Dispatch-Trailer on a protected branch + if: |- + ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')) + run: |- + echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch" + false + - name: Fail if Preprocessor-No-Write-Cache trailer is present for a scheduled workflow run + if: github.event.inputs.scheduled == 'true' + run: '! ./_scripts/noWriteCache.bash HEAD' + - if: github.repository != 'cue-lang/cuelang.org' + run: | + echo 'Setting CI_NO_SKIP_CACHE=true' + echo "CI_NO_SKIP_CACHE=true" >> $GITHUB_ENV + - if: runner.os == 'macOS' + run: |- + mkdir $HOME/.tmp + echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV + name: Set TMPDIR environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + mkdir -p ~/.lima/default + cat < ~/.lima/default/lima.yaml + mounts: + - location: "~" + writable: true + - location: "$TMPDIR" + writable: true + EOD + name: Write lima config (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + brew install colima docker + colima start --mount-type virtiofs + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + name: Install Docker (${{runner.os}}) + - if: runner.os == 'macOS' + run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV + name: Set DOCKER_HOST environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + name: Install macOS utils + run: brew install coreutils + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 20.9.0 + - name: Install Go + uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.0 + - name: Set common go env vars + run: |- + go env -w GOTOOLCHAIN=local + + # Dump env for good measure + go env + - if: runner.os == 'Linux' + name: Install Hugo (${{ runner.os }}) + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: 0.128.2 + extended: true + - if: runner.os == 'macOS' + name: Install Hugo (${{ runner.os }}) + run: brew install hugo + - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true' + run: | + if ./_scripts/noWriteCache.bash HEAD + then + echo 'Found Preprocessor-No-Write-Cache trailer' + echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV + fi + - name: Get go mod cache directory + id: go-mod-cache-dir + run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT} + - name: Get go build/test cache directory + id: go-cache-dir + run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT} + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + if: |- + (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + uses: actions/cache@v4 + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + uses: actions/cache/restore@v4 + if: |- + ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + - if: |- + github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test') + run: go clean -testcache + - name: Early git and code sanity checks + run: go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650 + - name: Perform early content checks + run: _scripts/contentLint.bash + - name: Populate CUE dependency cache + env: + CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }} + run: _scripts/cacheWarm.bash + - name: log into the central registry as porcuepine + run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }} + - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' + run: _scripts/tipPatchApply.bash + - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + env: + GOPRIVATE: cuelang.org/go + run: _scripts/tipUseAlternativeCUE.bash + - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + run: _scripts/regenPostInfraChange.bash + env: + GOPRIVATE: cuelang.org/go + - name: 'tip.cuelang.org: Deploy the site' + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + run: |- + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" + _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' + env: + GOPRIVATE: cuelang.org/go diff --git a/.github/workflows/trybot.yaml b/.github/workflows/trybot.yaml index 0bd61905c..4d38050c1 100644 --- a/.github/workflows/trybot.yaml +++ b/.github/workflows/trybot.yaml @@ -231,22 +231,3 @@ jobs: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }} ALGOLIA_INDEX_NAME: cuelang.org ALGOLIA_INDEX_FILE: ../_public/algolia.json - - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' - run: _scripts/tipPatchApply.bash - - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - env: - GOPRIVATE: cuelang.org/go - run: _scripts/tipUseAlternativeCUE.bash - - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - run: _scripts/regenPostInfraChange.bash - env: - GOPRIVATE: cuelang.org/go - - name: 'tip.cuelang.org: Deploy the site' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - run: |- - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' - env: - GOPRIVATE: cuelang.org/go diff --git a/internal/ci/github/tipdeploy.cue b/internal/ci/github/tipdeploy.cue new file mode 100644 index 000000000..6cdb12149 --- /dev/null +++ b/internal/ci/github/tipdeploy.cue @@ -0,0 +1,174 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "list" + "strings" + "strconv" + + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" + + "github.com/cue-lang/cuelang.org/internal/ci/netlify" +) + +workflows: tipdeploy: _repo.bashWorkflow & { + name: "tip deploy" + + on: { + push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches + + // Allow the trybots to run in response to a workflow_dispatch event. + // One source of workflow_dispatch events is the daily_tip_check + // workflow. + workflow_dispatch: { + // TODO: work out how to derive this "schema" from a CUE schema + inputs: { + scheduled: { + description: "Whether a workflow_dispatch was itself the result of a scheduled dispatch" + required: true + default: "false" // this is a string type field + } + } + } + } + + jobs: test: { + "runs-on": _repo.linuxMachine + + // We only want to run this workflow in the main repo + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" + + concurrency: { + group: "tip deploy" + // We do not set this to avoid getting failure messages for a cancel. + // "cancel-in-progress": true + } + + steps: [ + for v in _repo.checkoutCode {v}, + + for v in _installDockerMacOS {v}, + + _installMacOSUtils, + _setupBuildx, + _installNode, + for v in _installGo {v}, + _installHugoLinux, + _installHugoMacOS, + + // If the commit under test contains the trailer + // Preprocessor-No-Write-Cache: true, then set the + // PREPROCESSOR_NOWRITECACHE env var to non-empty. + githubactions.#Step & { + name: "Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true" + run: """ + if ./_scripts/noWriteCache.bash HEAD + then + echo 'Found Preprocessor-No-Write-Cache trailer' + echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV + fi + + """ + }, + + // cachePre must come after installing Node and Go, because the cache locations + // are established by running each tool. + for v in _setupGoActionsCaches {v}, + + // Run these early checks after we have restored the Go caches, + // as the checks are Go programs themselves. + _repo.earlyChecks, + + githubactions.#Step & { + name: "Perform early content checks" + run: "_scripts/contentLint.bash" + }, + + // This is the only step that needs to be given (read-only) access to the Central Registry. + // TODO: adopt any more specific command coming from https://cuelang.org/issue/3512. + // TODO: add cache dir to CI cache when it's visible via https://cuelang.org/issue/2838. + githubactions.#Step & { + name: "Populate CUE dependency cache" + env: CUE_TOKEN: "${{ secrets.NOTCUECKOO_CUE_TOKEN }}" + run: "_scripts/cacheWarm.bash" + }, + + // A number of pages that are part of cuelang.org require interacting + // with the Central Registry. These pages require users with slightly + // different access levels, in order to simulate (for example) private + // modules, with some users granted access whilst others are denied. + // The Central Registry has a special endpoint which generates access + // tokens for a set of such test user IDs. Access to this endpoint is + // sensitive, because in theory there is privilege escalation (even + // though in reality the test user IDs are intentionally not used + // for anything sensitive). As such, we use porcuepine (an account + // that is controlled by CUE Labs who also run the Central + // Registry) here in order to more carefully control in a CI + // environment who has access to this endpoint. + githubactions.#Step & { + name: "log into the central registry as porcuepine" + run: "go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}" + }, + + githubactions.#Step & { + name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue" + run: "_scripts/tipPatchApply.bash" + }, + + // npm install in hugo to allow serve test to pass + githubactions.#Step & { + run: "npm install" + "working-directory": "hugo" + }, + + githubactions.#Step & { + name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue" + // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) + // so that CLs aren't blocked by failures caused by unrelated changes. + + // Force Go to bypass the module proxy and sumdb for the + // cuelang.org/go module, ensuring that the absolute latest CUE + // pseudo-version is available to test against. + // + // TODO: is this really necessary? Tracking + // https://golang.org/issue/70042 to confirm. + env: GOPRIVATE: "cuelang.org/go" + + run: "_scripts/tipUseAlternativeCUE.bash" + }, + githubactions.#Step & { + name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue" + // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) + // so that CLs aren't blocked by failures caused by unrelated changes. + run: "_scripts/regenPostInfraChange.bash" + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" + }, + githubactions.#Step & { + name: "tip.cuelang.org: Deploy the site" + // Only run in the main repo on the default branch or its designated test branch. + run: """ + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" + _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' + """ + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" + }, + ] + } +} diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 4fb551e6c..689a05e3a 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -219,7 +219,7 @@ workflows: trybot: _repo.bashWorkflow & { // environment who has access to this endpoint. githubactions.#Step & { name: "log into the central registry as porcuepine" - run: "go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}" + run: "go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }}" }, _dist & { @@ -265,50 +265,6 @@ workflows: trybot: _repo.bashWorkflow & { ALGOLIA_INDEX_FILE: "../_public/algolia.json" } }, - - githubactions.#Step & { - name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue" - run: "_scripts/tipPatchApply.bash" - }, - - githubactions.#Step & { - name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - - // Force Go to bypass the module proxy and sumdb for the - // cuelang.org/go module, ensuring that the absolute latest CUE - // pseudo-version is available to test against. - // - // TODO: is this really necessary? Tracking - // https://golang.org/issue/70042 to confirm. - env: GOPRIVATE: "cuelang.org/go" - - run: "_scripts/tipUseAlternativeCUE.bash" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: "_scripts/regenPostInfraChange.bash" - - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Deploy the site" - // Only run in the main repo on the default branch or its designated test branch. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: """ - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" - _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' - """ - - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, ] }