From 7bf1822c490c384537751c8c31198ee6a50b5fc1 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Thu, 5 Sep 2024 13:22:33 +0100 Subject: [PATCH] internal/ci: re-baseline base from cue-lang/cue@82995bbb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This moves us to use the Central Registry (temporary namespace) definitions for GitHub Actions. We also copy the current pattern for "authenticating" a step that requires access to the Central Registry. Not the cleanest of patterns, but changing that does not belong in this CL. Signed-off-by: Paul Jolly Change-Id: I00b42fabdbec6dee8608974e86122724dce2fa9d Reviewed-on: https://review.gerrithub.io/c/cue-lang/vscode-cue/+/1200740 TryBot-Result: CUEcueckoo Reviewed-by: Daniel Martí --- .../{evict_caches.yml => evict_caches.yaml} | 4 +- ..._to_trybot.yml => push_tip_to_trybot.yaml} | 8 +-- .github/workflows/{trybot.yml => trybot.yaml} | 15 +++-- ...ybot_dispatch.yml => trybot_dispatch.yaml} | 2 +- cue.mod/module.cue | 8 ++- internal/ci/base/base.cue | 2 +- internal/ci/base/gerrithub.cue | 22 ++++---- internal/ci/base/github.cue | 56 ++++++++++++------- internal/ci/base/helpers.cue | 4 +- internal/ci/gen.go | 1 - internal/ci/github/logins.cue | 37 ++++++++++++ internal/ci/github/trybot.cue | 18 +++--- internal/ci/github/workflows.cue | 4 +- internal/ci/vendor/vendor_tool.cue | 50 ----------------- 14 files changed, 123 insertions(+), 108 deletions(-) rename .github/workflows/{evict_caches.yml => evict_caches.yaml} (98%) rename .github/workflows/{push_tip_to_trybot.yml => push_tip_to_trybot.yaml} (100%) rename .github/workflows/{trybot.yml => trybot.yaml} (95%) rename .github/workflows/{trybot_dispatch.yml => trybot_dispatch.yaml} (100%) create mode 100644 internal/ci/github/logins.cue delete mode 100644 internal/ci/vendor/vendor_tool.cue diff --git a/.github/workflows/evict_caches.yml b/.github/workflows/evict_caches.yaml similarity index 98% rename from .github/workflows/evict_caches.yml rename to .github/workflows/evict_caches.yaml index d87282c..f448788 100644 --- a/.github/workflows/evict_caches.yml +++ b/.github/workflows/evict_caches.yaml @@ -89,7 +89,7 @@ jobs: echo Latest commit: $sha echo "Trigger workflow on cue-lang/vscode-cue" - curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.CUECKOO_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail-with-body -X POST https://api.github.com/repos/cue-lang/vscode-cue/actions/workflows/trybot.yml/dispatches -d "{\"ref\":\"$j\"}" + curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.CUECKOO_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail-with-body -X POST https://api.github.com/repos/cue-lang/vscode-cue/actions/workflows/trybot.yaml/dispatches -d "{\"ref\":\"$j\"}" # Ensure that the trybot repo has the latest commit for # this branch. If the force-push results in a commit @@ -121,7 +121,7 @@ jobs: # We are up-to-date, i.e. the push did nothing, hence we need to trigger a workflow_dispatch # in the trybot repo. echo "Trigger workflow on cue-lang/vscode-cue-trybot" - curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.CUECKOO_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail-with-body -X POST https://api.github.com/repos/cue-lang/vscode-cue-trybot/actions/workflows/trybot.yml/dispatches -d "{\"ref\":\"$j\"}" + curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.CUECKOO_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail-with-body -X POST https://api.github.com/repos/cue-lang/vscode-cue-trybot/actions/workflows/trybot.yaml/dispatches -d "{\"ref\":\"$j\"}" else echo "Force-push to cue-lang/vscode-cue-trybot did work; nothing to do" fi diff --git a/.github/workflows/push_tip_to_trybot.yml b/.github/workflows/push_tip_to_trybot.yaml similarity index 100% rename from .github/workflows/push_tip_to_trybot.yml rename to .github/workflows/push_tip_to_trybot.yaml index aaa1829..edbad83 100644 --- a/.github/workflows/push_tip_to_trybot.yml +++ b/.github/workflows/push_tip_to_trybot.yaml @@ -9,10 +9,6 @@ concurrency: push_tip_to_trybot jobs: push: runs-on: ubuntu-22.04 - defaults: - run: - shell: bash - if: ${{github.repository == 'cue-lang/vscode-cue'}} steps: - name: Write netrc file for cueckoo Gerrithub run: |- @@ -48,3 +44,7 @@ jobs: echo "Giving up" exit 1 fi + defaults: + run: + shell: bash + if: ${{github.repository == 'cue-lang/vscode-cue'}} diff --git a/.github/workflows/trybot.yml b/.github/workflows/trybot.yaml similarity index 95% rename from .github/workflows/trybot.yml rename to .github/workflows/trybot.yaml index 91aa4d6..c14a143 100644 --- a/.github/workflows/trybot.yml +++ b/.github/workflows/trybot.yaml @@ -6,18 +6,18 @@ name: TryBot branches: - ci/test - master - pull_request: {} workflow_dispatch: {} + pull_request: {} jobs: test: runs-on: ubuntu-22.04 - defaults: - run: - shell: bash if: |- (contains(github.event.head_commit.message, ' Dispatch-Trailer: {"type":"trybot"')) || ! (contains(github.event.head_commit.message, ' Dispatch-Trailer: {"type":"')) + defaults: + run: + shell: bash steps: - name: Checkout code uses: actions/checkout@v4 @@ -101,7 +101,12 @@ jobs: - name: Verify run: go mod verify - name: Generate - run: go generate ./... + run: |- + export CUE_CONFIG_DIR=$(mktemp -d) + echo "$CUE_LOGINS" > $CUE_CONFIG_DIR/logins.json + go generate ./... + env: + CUE_LOGINS: ${{ secrets.NOTCUECKOO_CUE_LOGINS }} - name: Test run: go test ./... - name: Race test diff --git a/.github/workflows/trybot_dispatch.yml b/.github/workflows/trybot_dispatch.yaml similarity index 100% rename from .github/workflows/trybot_dispatch.yml rename to .github/workflows/trybot_dispatch.yaml index 923ae4d..a3d58bc 100644 --- a/.github/workflows/trybot_dispatch.yml +++ b/.github/workflows/trybot_dispatch.yaml @@ -9,10 +9,10 @@ name: Dispatch trybot jobs: trybot: runs-on: ubuntu-22.04 + if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }} defaults: run: shell: bash - if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }} steps: - name: Write netrc file for cueckoo Gerrithub run: |- diff --git a/cue.mod/module.cue b/cue.mod/module.cue index 6d645f9..56c8bef 100644 --- a/cue.mod/module.cue +++ b/cue.mod/module.cue @@ -1,4 +1,10 @@ module: "github.com/cue-lang/vscode-cue" language: { - version: "v0.10.0" + version: "v0.8.0" +} +deps: { + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions@v0": { + v: "v0.4.0" + default: true + } } diff --git a/internal/ci/base/base.cue b/internal/ci/base/base.cue index c0dea7d..5304b85 100644 --- a/internal/ci/base/base.cue +++ b/internal/ci/base/base.cue @@ -55,7 +55,7 @@ botGerritHubUser: *botGitHubUser | string botGerritHubUserPasswordSecretsKey: *(strings.ToUpper(botGitHubUser) + "_GERRITHUB_PASSWORD") | string botGerritHubUserEmail: *botGitHubUserEmail | string -workflowFileExtension: ".yml" +workflowFileExtension: ".yaml" linuxMachine: string diff --git a/internal/ci/base/gerrithub.cue b/internal/ci/base/gerrithub.cue index 0105ee0..5c2b551 100644 --- a/internal/ci/base/gerrithub.cue +++ b/internal/ci/base/gerrithub.cue @@ -3,15 +3,15 @@ package base // This file contains gerrithub related definitions etc import ( - encjson "encoding/json" + "encoding/json" "strings" - "github.com/SchemaStore/schemastore/src/schemas/json" + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" ) // trybotWorkflows is a template for trybot-based repos trybotWorkflows: { - (trybot.key): json.#Workflow & { + (trybot.key): githubactions.#Workflow & { on: workflow_dispatch: {} } "\(trybot.key)_dispatch": trybotDispatchWorkflow @@ -45,7 +45,7 @@ trybotDispatchWorkflow: bashWorkflow & { (trybot.key): { "runs-on": linuxMachine - let goodDummyData = [if encjson.Marshal(#dummyDispatch) != _|_ {true}, false][0] + let goodDummyData = [if json.Marshal(#dummyDispatch) != _|_ {true}, false][0] // We set the "on" conditions above, but this would otherwise mean we // run for all dispatch events. @@ -68,7 +68,7 @@ trybotDispatchWorkflow: bashWorkflow & { steps: [ writeNetrcFile, - json.#step & { + githubactions.#Step & { name: "Write fake payload" id: "payload" if: "github.repository == '\(githubRepositoryPath)' && \(isTestDefaultBranch)" @@ -80,7 +80,7 @@ trybotDispatchWorkflow: bashWorkflow & { run: #""" cat <> $GITHUB_OUTPUT value< ~/.netrc diff --git a/internal/ci/base/github.cue b/internal/ci/base/github.cue index 5b1c4ce..e13296b 100644 --- a/internal/ci/base/github.cue +++ b/internal/ci/base/github.cue @@ -3,20 +3,20 @@ package base // This file contains aspects principally related to GitHub workflows import ( - encjson "encoding/json" + "encoding/json" "list" "strings" "strconv" - "github.com/SchemaStore/schemastore/src/schemas/json" + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" ) -bashWorkflow: json.#Workflow & { +bashWorkflow: githubactions.#Workflow & { jobs: [string]: defaults: run: shell: "bash" } installGo: { - #setupGo: json.#step & { + #setupGo: githubactions.#Step & { name: "Install Go" uses: "actions/setup-go@v5" with: { @@ -49,7 +49,7 @@ installGo: { #setupGo, { - json.#step & { + githubactions.#Step & { name: "Set common go env vars" run: """ go env -w GOTOOLCHAIN=local @@ -63,7 +63,7 @@ installGo: { } checkoutCode: { - #actionsCheckout: json.#step & { + #actionsCheckout: githubactions.#Step & { name: "Checkout code" uses: "actions/checkout@v4" @@ -90,17 +90,17 @@ checkoutCode: { // per the bug report at https://github.com/MestreLion/git-tools/issues/47, // so we first reset all directory timestamps to a static time as a fallback. // TODO(mvdan): May be unnecessary once the Go bug above is fixed. - json.#step & { + githubactions.#Step & { name: "Reset git directory modification times" run: "touch -t 202211302355 $(find * -type d)" }, - json.#step & { + githubactions.#Step & { name: "Restore git file modification times" uses: "chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe" }, { - json.#step & { + githubactions.#Step & { name: "Try to extract \(dispatchTrailer)" id: dispatchTrailerStepID run: """ @@ -124,7 +124,7 @@ checkoutCode: { // Safety nets to flag if we ever have a Dispatch-Trailer slip through the // net and make it to master - json.#step & { + githubactions.#Step & { name: "Check we don't have \(dispatchTrailer) on a protected branch" if: "\(isProtectedBranch) && \(containsDispatchTrailer)" run: """ @@ -135,9 +135,9 @@ checkoutCode: { ] } -earlyChecks: json.#step & { +earlyChecks: githubactions.#Step & { name: "Early git and code sanity checks" - run: "go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650" + run: *"go run cuelang.org/go/internal/ci/checks@v0.11.0-0.dev.0.20240903133435-46fb300df650" | string } curlGitHubAPI: { @@ -177,7 +177,7 @@ setupGoActionsCaches: { let cacheRestoreKeys = "\(#os)-\(#goVersion)" - let cacheStep = json.#step & { + let cacheStep = githubactions.#Step & { with: { path: strings.Join(cacheDirs, "\n") @@ -197,12 +197,12 @@ setupGoActionsCaches: { [ // TODO: once https://github.com/actions/setup-go/issues/54 is fixed, // we could use `go env` outputs from the setup-go step. - json.#step & { + githubactions.#Step & { name: "Get go mod cache directory" id: goModCacheDirID run: #"echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}"# }, - json.#step & { + githubactions.#Step & { name: "Get go build/test cache directory" id: goCacheDirID run: #"echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}"# @@ -240,7 +240,7 @@ setupGoActionsCaches: { // // Critically we only want to do this in the main repo, not the trybot // repo. - json.#step & { + githubactions.#Step & { if: "github.repository == '\(githubRepositoryPath)' && (\(isProtectedBranch) || github.ref == 'refs/heads/\(testDefaultBranch)')" run: "go clean -testcache" } @@ -270,13 +270,13 @@ isReleaseTag: { (_matchPattern & {variable: "github.ref", pattern: "refs/tags/\(releaseTagPattern)"}).expr } -checkGitClean: json.#step & { +checkGitClean: githubactions.#Step & { name: "Check that git is clean at the end of the job" if: "always()" run: "test -z \"$(git status --porcelain)\" || (git status; git diff; false)" } -repositoryDispatch: json.#step & { +repositoryDispatch: githubactions.#Step & { #githubRepositoryPath: *githubRepositoryPath | string #botGitHubUserTokenSecretsKey: *botGitHubUserTokenSecretsKey | string #arg: _ @@ -285,7 +285,25 @@ repositoryDispatch: json.#step & { name: string run: #""" - \#(_curlGitHubAPI) --fail --request POST --data-binary \#(strconv.Quote(encjson.Marshal(#arg))) https://api.github.com/repos/\#(#githubRepositoryPath)/dispatches + \#(_curlGitHubAPI) --fail --request POST --data-binary \#(strconv.Quote(json.Marshal(#arg))) https://api.github.com/repos/\#(#githubRepositoryPath)/dispatches + """# +} + +workflowDispatch: githubactions.#Step & { + #githubRepositoryPath: *githubRepositoryPath | string + #botGitHubUserTokenSecretsKey: *botGitHubUserTokenSecretsKey | string + #workflowID: string + + // params are defined per https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event + #params: *{ + ref: defaultBranch + } | _ + + _curlGitHubAPI: curlGitHubAPI & {#tokenSecretsKey: #botGitHubUserTokenSecretsKey, _} + + name: string + run: #""" + \#(_curlGitHubAPI) --fail --request POST --data-binary \#(strconv.Quote(json.Marshal(#params))) https://api.github.com/repos/\#(#githubRepositoryPath)/actions/workflows/\#(#workflowID)/dispatches """# } diff --git a/internal/ci/base/helpers.cue b/internal/ci/base/helpers.cue index 31321a7..027b461 100644 --- a/internal/ci/base/helpers.cue +++ b/internal/ci/base/helpers.cue @@ -14,8 +14,8 @@ import ( _matchPattern: { variable: string pattern: string - expr: [ - if strings.HasSuffix(pattern, "*") { + expr: [ + if strings.HasSuffix(pattern, "*") { let prefix = strings.TrimSuffix(pattern, "*") "startsWith(\(variable), '\(prefix)')" }, diff --git a/internal/ci/gen.go b/internal/ci/gen.go index 3e44d79..a4baee6 100644 --- a/internal/ci/gen.go +++ b/internal/ci/gen.go @@ -14,5 +14,4 @@ package ci -//go:generate cue cmd importjsonschema ./vendor //go:generate cue cmd gen diff --git a/internal/ci/github/logins.cue b/internal/ci/github/logins.cue new file mode 100644 index 0000000..7756817 --- /dev/null +++ b/internal/ci/github/logins.cue @@ -0,0 +1,37 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" +) + +// _registryReadOnlyAccessStep defines a step that configures +// a read-only Central Registry access credential. The actual +// command should be placed in the _run field. +_registryReadOnlyAccessStep: githubactions.#Step & { + _run!: string + env: { + // Note: this token has read-only access to the registry + // and is used only because we need some credentials + // to pull dependencies from the Central Registry. + CUE_LOGINS: "${{ secrets.NOTCUECKOO_CUE_LOGINS }}" + } + run: """ + export CUE_CONFIG_DIR=$(mktemp -d) + echo "$CUE_LOGINS" > $CUE_CONFIG_DIR/logins.json + \(_run) + """ +} diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 009185e..5126a9f 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -17,7 +17,7 @@ package github import ( "list" - "github.com/SchemaStore/schemastore/src/schemas/json" + "github.com/cue-tmp/jsonschema-pub/exp1/githubactions" ) // The trybot workflow. @@ -59,27 +59,27 @@ workflows: trybot: _repo.bashWorkflow & { for v in _setupGoActionsCaches {v}, - json.#step & { + githubactions.#Step & { name: "Verify" run: "go mod verify" }, - json.#step & { + _registryReadOnlyAccessStep & { name: "Generate" - run: "go generate ./..." + _run: "go generate ./..." }, - json.#step & { + githubactions.#Step & { name: "Test" run: "go test ./..." }, - json.#step & { + githubactions.#Step & { name: "Race test" run: "go test -race ./..." }, - json.#step & { + githubactions.#Step & { name: "staticcheck" run: "go run honnef.co/go/tools/cmd/staticcheck@v0.5.1 ./..." }, - json.#step & { + githubactions.#Step & { name: "Tidy" run: "go mod tidy" }, @@ -88,7 +88,7 @@ workflows: trybot: _repo.bashWorkflow & { } } -_installCUE: json.#step & { +_installCUE: githubactions.#Step & { name: "Install CUE" uses: "cue-lang/setup-cue@v1.0.1" with: version: "v0.11.0-rc.1" diff --git a/internal/ci/github/workflows.cue b/internal/ci/github/workflows.cue index 23d3380..d7a0527 100644 --- a/internal/ci/github/workflows.cue +++ b/internal/ci/github/workflows.cue @@ -42,9 +42,9 @@ workflows: close({ _repo.trybotWorkflows trybot: _ - trybot_dispatch: #dummyDispatch: dummyDispatch + trybot_dispatch: #dummyDispatch: _dummyDispatch }) -dummyDispatch: _repo.#dispatch & { +_dummyDispatch: _repo.#dispatch & { type: _repo.trybot.key } diff --git a/internal/ci/vendor/vendor_tool.cue b/internal/ci/vendor/vendor_tool.cue deleted file mode 100644 index 74ff3c3..0000000 --- a/internal/ci/vendor/vendor_tool.cue +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2021 The CUE Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vendor - -import ( - "path" - - "tool/exec" - "tool/http" -) - -// _cueCmd defines the command that is run to run cmd/cue. -// This is factored out in order that the cue-github-actions -// project which "vendors" the various workflow-related -// packages can specify "cue" as the value so that unity -// tests can specify the cmd/cue binary to use. -_cueCmd: string | *"go run cuelang.org/go/cmd/cue@v0.10.0" @tag(cue_cmd) - -// For the commands below, note we use simple yet hacky path resolution, rather -// than anything that might derive the module root using go list or similar, in -// order that we have zero dependencies. - -// importjsonschema vendors a CUE-imported version of the JSONSchema that -// defines GitHub workflows into the main module's cue.mod/pkg. -command: importjsonschema: { - getJSONSchema: http.Get & { - request: body: "" - - // Tip link for humans: - // https://github.com/SchemaStore/schemastore/blob/master/src/schemas/json/github-workflow.json - url: "https://raw.githubusercontent.com/SchemaStore/schemastore/88d26ad0d451cbd5ebc70218062850aa905bdf18/src/schemas/json/github-workflow.json" - } - import: exec.Run & { - _outpath: path.FromSlash("../../cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue", "unix") - stdin: getJSONSchema.response.body - cmd: "\(_cueCmd) import -f -p json -l #Workflow: -o \(_outpath) jsonschema: -" - } -}