diff --git a/.github/workflows/tipdeploy.yaml b/.github/workflows/tipdeploy.yaml new file mode 100644 index 000000000..d073cd0f2 --- /dev/null +++ b/.github/workflows/tipdeploy.yaml @@ -0,0 +1,182 @@ +# Code generated internal/ci/ci_tool.cue; DO NOT EDIT. + +name: tip deploy +"on": + push: + branches: + - ci/test + - master + workflow_dispatch: {} +concurrency: + group: tip deploy + cancel-in-progress: false +jobs: + test: + runs-on: ubuntu-22.04 + if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Reset git directory modification times + run: touch -t 202211302355 $(find * -type d) + - name: Restore git file modification times + uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe + - name: Try to extract Dispatch-Trailer + id: DispatchTrailer + run: |- + x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')" + if [[ "$x" == "" ]] + then + # Some steps rely on the presence or otherwise of the Dispatch-Trailer. + # We know that we don't have a Dispatch-Trailer in this situation, + # hence we use the JSON value null in order to represent that state. + # This means that GitHub expressions can determine whether a Dispatch-Trailer + # is present or not by checking whether the fromJSON() result of the + # output from this step is the JSON value null or not. + x=null + fi + echo "value<> $GITHUB_OUTPUT + echo "$x" >> $GITHUB_OUTPUT + echo "EOD" >> $GITHUB_OUTPUT + - name: Check we don't have Dispatch-Trailer on a protected branch + if: |- + ((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')) + run: |- + echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch" + false + - if: runner.os == 'macOS' + run: |- + mkdir $HOME/.tmp + echo "TMPDIR=$HOME/.tmp" >> $GITHUB_ENV + name: Set TMPDIR environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + mkdir -p ~/.lima/default + cat < ~/.lima/default/lima.yaml + mounts: + - location: "~" + writable: true + - location: "$TMPDIR" + writable: true + EOD + name: Write lima config (${{runner.os}}) + - if: runner.os == 'macOS' + run: |- + brew install colima docker + colima start --mount-type virtiofs + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + name: Install Docker (${{runner.os}}) + - if: runner.os == 'macOS' + run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> $GITHUB_ENV + name: Set DOCKER_HOST environment variable (${{runner.os}}) + - if: runner.os == 'macOS' + name: Install macOS utils + run: brew install coreutils + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 20.9.0 + - name: Install Go + uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.0 + - name: Set common go env vars + run: |- + go env -w GOTOOLCHAIN=local + + # Dump env for good measure + go env + - if: runner.os == 'Linux' + name: Install Hugo (${{ runner.os }}) + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: 0.128.2 + extended: true + - if: runner.os == 'macOS' + name: Install Hugo (${{ runner.os }}) + run: brew install hugo + - name: 'Set PREPROCESSOR_NOWRITECACHE if Preprocessor-No-Write-Cache: true' + run: | + if ./_scripts/noWriteCache.bash HEAD + then + echo 'Found Preprocessor-No-Write-Cache trailer' + echo "PREPROCESSOR_NOWRITECACHE=true" >> $GITHUB_ENV + fi + - name: Get go mod cache directory + id: go-mod-cache-dir + run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT} + - name: Get go build/test cache directory + id: go-cache-dir + run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT} + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + if: |- + (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + uses: actions/cache@v4 + - with: + path: |- + ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download + ${{ steps.go-cache-dir.outputs.dir }} + ~/.cache/dockercache + ~/.cache/node-gyp + ~/.npm + ${{ github.workspace }}/playground/.webpack_cache + key: ${{ runner.os }}-1.23.0-${{ github.run_id }} + restore-keys: ${{ runner.os }}-1.23.0 + uses: actions/cache/restore@v4 + if: |- + ! (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) + - if: |- + github.repository == 'cue-lang/cuelang.org' && (((github.ref == 'refs/heads/master') && (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test') + run: go clean -testcache + - name: Early git and code sanity checks + run: go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650 + - name: Perform early content checks + run: _scripts/contentLint.bash + - name: Populate CUE dependency cache + env: + CUE_TOKEN: ${{ secrets.NOTCUECKOO_CUE_TOKEN }} + run: _scripts/cacheWarm.bash + - name: log into the central registry as porcuepine + run: go run cuelang.org/go/cmd/cue login --token ${{ secrets.PORCUEPINE_CUE_TOKEN }} + - name: Patch the site to be compatible with the tip of cue-lang/cue + run: _scripts/tipPatchApply.bash + - name: npm install in hugo dir + run: npm install + working-directory: hugo + - name: Configure the site to use the tip of cue-lang/cue + env: + GOPRIVATE: cuelang.org/go + run: _scripts/tipUseAlternativeCUE.bash + - name: Build the site against the tip of cue-lang/cue + run: _scripts/regenPostInfraChange.bash + env: + GOPRIVATE: cuelang.org/go + - name: Deploy the site + run: |- + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" + _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' + env: + GOPRIVATE: cuelang.org/go diff --git a/.github/workflows/trybot.yaml b/.github/workflows/trybot.yaml index 0bd61905c..9dba27404 100644 --- a/.github/workflows/trybot.yaml +++ b/.github/workflows/trybot.yaml @@ -184,7 +184,8 @@ jobs: if: always() run: test -z "$(git status --porcelain)" || (git status; git diff; false) - run: ./_scripts/buildDockerImage.bash - - run: npm install + - name: npm install in hugo dir + run: npm install working-directory: hugo - name: Test run: go test ./... @@ -231,22 +232,19 @@ jobs: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }} ALGOLIA_INDEX_NAME: cuelang.org ALGOLIA_INDEX_FILE: ../_public/algolia.json - - name: 'tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue' + - name: Patch the site to be compatible with the tip of cue-lang/cue run: _scripts/tipPatchApply.bash - - name: 'tip.cuelang.org: Configure the site to use the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: npm install in hugo dir + run: npm install + working-directory: hugo + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: Configure the site to use the tip of cue-lang/cue env: GOPRIVATE: cuelang.org/go run: _scripts/tipUseAlternativeCUE.bash - - name: 'tip.cuelang.org: Build the site against the tip of cue-lang/cue' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) + name: Build the site against the tip of cue-lang/cue run: _scripts/regenPostInfraChange.bash env: GOPRIVATE: cuelang.org/go - - name: 'tip.cuelang.org: Deploy the site' - if: github.repository == 'cue-lang/cuelang.org' && (github.ref == 'refs/heads/master' || (github.ref == 'refs/heads/ci/test')) - run: |- - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - _scripts/tipDeploy.bash 'cueckoo' 'cueckoo@gmail.com' - env: - GOPRIVATE: cuelang.org/go diff --git a/internal/ci/github/tipdeploy.cue b/internal/ci/github/tipdeploy.cue new file mode 100644 index 000000000..d51d92bab --- /dev/null +++ b/internal/ci/github/tipdeploy.cue @@ -0,0 +1,132 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "list" + + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" +) + +// tipdeploy is responsible for publishing a new version of tip.cuelang.org. +// +// Much of this workflow follows the steps in the trybot workflow. +workflows: tipdeploy: _repo.bashWorkflow & { + name: "tip deploy" + + on: { + // Allow this workflow to run in response to a new commit at the tip of + // this repo. We also allow a tip deploy to be triggered from a push to + // ci/test to help with testing CLs. + push: branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches + + // Allow this workflow to run in response to a workflow_dispatch event, + // specifically the event of a new commit at the tip of CUE needs to + // trigger a new tip deploy. + workflow_dispatch: {} + } + + // We limit this workflow to run with parallelism of 1. This should + // prevent us hitting races that occur when multiple commits land "at the + // same time" in the main CUE repo, each triggering a tip deploy. + concurrency: { + group: "tip deploy" + // We do not set cancel-in-progress to true to avoid getting failure + // messages for a cancel, per the following GitHub bug: + // + // https://github.com/orgs/community/discussions/50725 + // + "cancel-in-progress": false + } + + jobs: test: { + "runs-on": _repo.linuxMachine + + // We only want to run this workflow in the main repo + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" + + steps: [ + for v in _repo.checkoutCode {v}, + + for v in _installDockerMacOS {v}, + _installMacOSUtils, + _setupBuildx, + _installNode, + for v in _installGo {v}, + _installHugoLinux, + _installHugoMacOS, + + _setNoWriteCache, + + for v in _setupGoActionsCaches {v}, + + _repo.earlyChecks, + + _contentLint, + + _cacheWarm, + + _porcuepineCueLogin, + + _applyTipPatches, + + // npm install in hugo to allow serve test to pass which is run as + // part of regenPostInfraChange.bash + _npmInstall, + + _useTipOfCUE, + + _regenPostInfraChange, + + { + name: "Deploy the site" + run: """ + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" + _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' + """ + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" + }, + ] + } +} + +_applyTipPatches: githubactions.#Step & { + name: "Patch the site to be compatible with the tip of cue-lang/cue" + run: "_scripts/tipPatchApply.bash" +} + +_useTipOfCUE: githubactions.#Step & { + name: "Configure the site to use the tip of cue-lang/cue" + + // Force Go to bypass the module proxy and sumdb for the + // cuelang.org/go module, ensuring that the absolute latest CUE + // pseudo-version is available to test against. + // + // TODO: is the use of GOPRIVATE (as opposed to GONOPROXY) really + // necessary? Tracking https://golang.org/issue/70042 to confirm. + env: GOPRIVATE: "cuelang.org/go" + + run: "_scripts/tipUseAlternativeCUE.bash" +} + +_regenPostInfraChange: githubactions.#Step & { + name: "Build the site against the tip of cue-lang/cue" + run: "_scripts/regenPostInfraChange.bash" + + // TODO: See comment in previous step + env: GOPRIVATE: "cuelang.org/go" +} diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 2422ddb58..8bc8fb030 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -222,49 +222,17 @@ workflows: trybot: _repo.bashWorkflow & { } }, - githubactions.#Step & { - name: "tip.cuelang.org: Patch the site to be compatible with the tip of cue-lang/cue" - run: "_scripts/tipPatchApply.bash" - }, + // We do this on all branches to catch changes in CL that cause the + // application of tip patches to fail. This doesn't guarantee later + // success, but it's a useful early indicator. + _applyTipPatches, - githubactions.#Step & { - name: "tip.cuelang.org: Configure the site to use the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - - // Force Go to bypass the module proxy and sumdb for the - // cuelang.org/go module, ensuring that the absolute latest CUE - // pseudo-version is available to test against. - // - // TODO: is this really necessary? Tracking - // https://golang.org/issue/70042 to confirm. - env: GOPRIVATE: "cuelang.org/go" - - run: "_scripts/tipUseAlternativeCUE.bash" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Build the site against the tip of cue-lang/cue" - // Only run in the main repo on the default branch or its designated test branch (i.e not CLs) - // so that CLs aren't blocked by failures caused by unrelated changes. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: "_scripts/regenPostInfraChange.bash" - - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, - githubactions.#Step & { - name: "tip.cuelang.org: Deploy the site" - // Only run in the main repo on the default branch or its designated test branch. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" - run: """ - git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(_repo.botGitHubUser):${{ secrets.\(_repo.botGitHubUserTokenSecretsKey) }} | base64)" - _scripts/tipDeploy.bash '\(_repo.botGitHubUser)' '\(_repo.botGitHubUserEmail)' - """ + // npm install post applying of tip patches in case there is an relevant patch + _mainRepoDefaulBranch & _npmInstall, - // TODO: See comment in previous step - env: GOPRIVATE: "cuelang.org/go" - }, + _mainRepoDefaulBranch & _useTipOfCUE, + + _mainRepoDefaulBranch & _regenPostInfraChange, ] } @@ -286,6 +254,10 @@ workflows: trybot: _repo.bashWorkflow & { } } +_mainRepoDefaulBranch: githubactions.#Step & { + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.defaultBranch)' || \(_repo.isTestDefaultBranch))" +} + _installNode: githubactions.#Step & { name: "Install Node" uses: "actions/setup-node@v4" @@ -488,6 +460,7 @@ _porcuepineCueLogin: githubactions.#Step & { // npm install in hugo to allow serve test to pass _npmInstall: githubactions.#Step & { + name: "npm install in hugo dir" run: "npm install" "working-directory": "hugo" }