From a662ad854373692cab2c7d14338485a3860fb1ef Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Sat, 17 Feb 2024 06:02:20 +0000 Subject: [PATCH] preprocessor: install multiple versions of CUE in docker image Currently we only install a single CUE version in the default site docker image: v0.8.0-alpha.1. However, not all docs on the site require installing a pre-release version. Update the default image to install multiple versions under /cues. We drive that installation by configuration. In the root of the site configuration there is already a 'versions' field used to drive various tool versions used in the base image and elsewhere. The 'cue' field of the 'versions' field is now a struct, mapping a human name to a CUE version: cue: { latest: "v0.7.1" prerelease: "v0.8.0-alpha.1" tip: "v0.8.0-alpha.1" } This configuration is used to update the Dockerfile template. As a bit of a hack (for now) we use the prerelease version of CUE as the default in the default image. For guides that require an alternative version, they can update PATH using a hidden script at the top of the guide. In such cases it is good practice to then have a regular script run to show 'cue version' so the reader is clear on the required version for that guide. We also update the PATH that is explicitly set in the generated bash file. The TODO above that hack still holds; it is unclear why the environment established in the Dockfile and entrypoint is not "good" at the point of running our script. The configuration map of CUE versions is not fully utilised in this CL. Rather, at a later date, we will drive more of the site, labels etc based on these versions, effectively migrating from the previous (current) state of hanging everything off a single version. Those changes will happen over time in subsequent CLs. Another important usage of the configured CUE versions is in guides that use the Go API. To make such guides reproducible, we 'go get' a specific version of cuelang.org/go rather, that '@latest'. This requires us to run a reader-visible command such as: go get cuelang.org/go@v0.7.1 Rather than hard-code such a version, we want to use the CUE versions configured at the site level, described above. Currently we have no mechanism of referring from a script block to that configuration. This CL introduces a basic initial version that uses specially named environment variables: {{{with script "en" "example"}}} go get cuelang.org/go@$CUELANG_CUE_PRERELEASE {{{end}}} The environment variables follow a pattern CUELANG_CUE_$NAME where $NAME refers to the human readable field name in the 'cue' struct described above. The substitution of these environment variables happens after the parsing of script nodes, but prior to the running of the multi-step script. This ensures that the actual command run includes the resolved version. It also ensures that the rendered Hugo page includes the resolved version, as opposed to an obscurely named environment variable. The substitution of these environment variables happens via an adapted version of os.Expand that conditionally substitutes environment variables. This code is vendored alongside the preprocessor code with a couple of tests to cover the extended functionality. Preprocessor-No-Write-Cache: true Signed-off-by: Paul Jolly Change-Id: Ie8f2da1cac137ce8c390b5c6d08dcb449c4ff78a Dispatch-Trailer: {"type":"trybot","CL":1177003,"patchset":8,"ref":"refs/changes/03/1177003/8","targetBranch":"alpha"} --- hugo/config/_default/params.toml | 2 + .../cmd/preprocessor/cmd/_docker/Dockerfile | 27 ++++- internal/cmd/preprocessor/cmd/execute.go | 17 +++ .../cmd/preprocessor/cmd/execute_context.go | 15 +++ internal/cmd/preprocessor/cmd/expand.go | 104 ++++++++++++++++++ internal/cmd/preprocessor/cmd/expand_test.go | 65 +++++++++++ .../preprocessor/cmd/gen_dockerimagetag.go | 2 +- internal/cmd/preprocessor/cmd/rootfile.go | 2 +- internal/cmd/preprocessor/cmd/script_node.go | 11 +- .../testdata/execute_multistagescript.txtar | 33 +++++- .../src/config/gen_cuelang_org_go_version.ts | 1 + site.cue | 50 +++++++-- 12 files changed, 306 insertions(+), 23 deletions(-) create mode 100644 internal/cmd/preprocessor/cmd/expand.go create mode 100644 internal/cmd/preprocessor/cmd/expand_test.go diff --git a/hugo/config/_default/params.toml b/hugo/config/_default/params.toml index 7b741f8bd..b40a7e449 100644 --- a/hugo/config/_default/params.toml +++ b/hugo/config/_default/params.toml @@ -18,6 +18,8 @@ logo = "svg/logo-alpha.svg" # Contents allows for markdown, leave out the button if you don't want a button [notification] type = 'test' + + # TODO: move away from defaulting to the pre-release of CUE content = '**Note:** documentation on this site relies on CUE v0.8.0-alpha.1' [notification.button] link = 'https://github.com/cue-lang/cue/releases' diff --git a/internal/cmd/preprocessor/cmd/_docker/Dockerfile b/internal/cmd/preprocessor/cmd/_docker/Dockerfile index 7189a1727..2ed2dfa6b 100644 --- a/internal/cmd/preprocessor/cmd/_docker/Dockerfile +++ b/internal/cmd/preprocessor/cmd/_docker/Dockerfile @@ -15,18 +15,37 @@ RUN \ export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ go install -trimpath github.com/rogpeppe/go-internal/cmd/testscript@v1.11.0 +RUN mkdir /cues + +RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/latest go install -trimpath cuelang.org/go/cmd/cue@v0.7.1 + RUN \ --mount=type=cache,target=/cache/gocache \ --mount=type=cache,target=/cache/gomodcache \ export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ - go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 + GOBIN=/cues/prerelease go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 + +RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/tip go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 FROM golang:1.22.0 RUN mkdir -p /go/bin +# TODO: move away from defaulting to the pre-release of CUE +ENV PATH="/cues/prerelease:${PATH}" + ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" -ENV CUE_VERSION="v0.8.0-alpha.1" +ENV CUELANG_CUE_LATEST="v0.7.1" +ENV CUELANG_CUE_PRERELEASE="v0.8.0-alpha.1" +ENV CUELANG_CUE_TIP="v0.8.0-alpha.1" WORKDIR / @@ -36,6 +55,8 @@ RUN chmod 755 /usr/bin/entrypoint.sh RUN chown root:root /usr/bin/entrypoint.sh COPY --from=build /go/bin/testscript /go/bin -COPY --from=build /go/bin/cue /go/bin +COPY --from=build /cues/latest/cue /cues/latest/cue +COPY --from=build /cues/prerelease/cue /cues/prerelease/cue +COPY --from=build /cues/tip/cue /cues/tip/cue ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/internal/cmd/preprocessor/cmd/execute.go b/internal/cmd/preprocessor/cmd/execute.go index 9c018370d..8600378be 100644 --- a/internal/cmd/preprocessor/cmd/execute.go +++ b/internal/cmd/preprocessor/cmd/execute.go @@ -134,6 +134,23 @@ type executionContext struct { // preprocessor. config cue.Value + // cueVersions is a map of name to version of CUE versions configured on the + // site. e.g. + // + // latest => v0.7.1 + // prerelease => v0.8.0-alpha.1 + cueVersions map[string]string + + // cueEnvVersions is a map of env var name to CUE versions, derived from + // cueVersions. For example the entry: + // + // latest => v0.7.0 + // + // in cueVersions exists in this map as: + // + // CUELANG_CUE_LATEST => v0.7.0 + cueEnvVersions map[string]string + // selfHash is the hash of the preprocessor itself that hashing calculations // should use if they need the preprocessor to affect the hash result. selfHash string diff --git a/internal/cmd/preprocessor/cmd/execute_context.go b/internal/cmd/preprocessor/cmd/execute_context.go index c2f24c929..b01f1a367 100644 --- a/internal/cmd/preprocessor/cmd/execute_context.go +++ b/internal/cmd/preprocessor/cmd/execute_context.go @@ -117,6 +117,21 @@ func (ec *executeContext) execute() error { } ec.config = v + // Load the CUE versions configured as part of the site + ec.cueVersions = make(map[string]string) + ec.cueEnvVersions = make(map[string]string) + cueVersionsPath := cue.ParsePath("versions.cue") + cueVersions := ec.config.LookupPath(cueVersionsPath) + if cueVersions.Exists() { + if err := cueVersions.Decode(&ec.cueVersions); err != nil { + return ec.errorf("%v: failed to decode %v: %v", cueVersionsPath, err) + } + for k, v := range ec.cueVersions { + key := fmt.Sprintf("CUELANG_CUE_%s", strings.ToUpper(k)) + ec.cueEnvVersions[key] = v + } + } + // Now load config per page for _, d := range ec.order { p := ec.pages[d] diff --git a/internal/cmd/preprocessor/cmd/expand.go b/internal/cmd/preprocessor/cmd/expand.go new file mode 100644 index 000000000..d49b2b67e --- /dev/null +++ b/internal/cmd/preprocessor/cmd/expand.go @@ -0,0 +1,104 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +// The code below is copied and modified from the Go source as of +// 9cc0f9cba2f2db97f5ba6c2c482bafaaa34c0381 +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// expand replaces ${var} or $var in the string based on the mapping function +// if the mapping function returns true. +func expand(s string, mapping func(string) (bool, string)) string { + var buf []byte + // ${} is all ASCII, so bytes are fine for this operation. + i := 0 + for j := 0; j < len(s); j++ { + if s[j] == '$' && j+1 < len(s) { + if buf == nil { + buf = make([]byte, 0, 2*len(s)) + } + buf = append(buf, s[i:j]...) + name, w := getShellName(s[j+1:]) + if name == "" && w > 0 { + // Encountered invalid syntax; eat the + // characters. + } else if name == "" { + // Valid syntax, but $ was not followed by a + // name. Leave the dollar character untouched. + buf = append(buf, s[j]) + } else { + mapped, repl := mapping(name) + if mapped { + buf = append(buf, repl...) + } else { + buf = append(buf, s[j:j+1+w]...) + } + } + j += w + i = j + 1 + } + } + if buf == nil { + return s + } + return string(buf) + s[i:] +} + +// isShellSpecialVar reports whether the character identifies a special +// shell variable such as $*. +func isShellSpecialVar(c uint8) bool { + switch c { + case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore. +func isAlphaNum(c uint8) bool { + return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' +} + +// getShellName returns the name that begins the string and the number of bytes +// consumed to extract it. If the name is enclosed in {}, it's part of a ${} +// expansion and two more bytes are needed than the length of the name. +func getShellName(s string) (string, int) { + switch { + case s[0] == '{': + if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' { + return s[1:2], 3 + } + // Scan to closing brace + for i := 1; i < len(s); i++ { + if s[i] == '}' { + if i == 1 { + return "", 2 // Bad syntax; eat "${}" + } + return s[1:i], i + 1 + } + } + return "", 1 // Bad syntax; eat "${" + case isShellSpecialVar(s[0]): + return s[0:1], 1 + } + // Scan alphanumerics. + var i int + for i = 0; i < len(s) && isAlphaNum(s[i]); i++ { + } + return s[:i], i +} diff --git a/internal/cmd/preprocessor/cmd/expand_test.go b/internal/cmd/preprocessor/cmd/expand_test.go new file mode 100644 index 000000000..b78c3a15e --- /dev/null +++ b/internal/cmd/preprocessor/cmd/expand_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "testing" + + "github.com/go-quicktest/qt" +) + +func TestExpand(t *testing.T) { + mapping := func(s string) (bool, string) { + switch s { + case "BANANA": + return true, "apple" + default: + return false, "" + } + } + cases := []struct { + name string + input string + want string + }{ + { + name: "entire string", + input: "$BANANA", + want: "apple", + }, + { + name: "double dollar", // nothing replace + input: "$$BANANA", + want: "$$BANANA", + }, + { + name: "braces", + input: "${BANANA}", + want: "apple", + }, + { + name: "another var", // left untouched + input: "$SOMETHING", + want: "$SOMETHING", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := expand(c.input, mapping) + qt.Assert(t, qt.Equals(got, c.want)) + }) + } +} diff --git a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go index 59a442b40..c220b9e48 100644 --- a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go +++ b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go @@ -2,4 +2,4 @@ package cmd -const dockerImageTag = "preprocessor:6e7cb00a0a5ab45565a39b9daa398f8fa8e2faa58a7b3baae3101c59a43764b2" +const dockerImageTag = "preprocessor:fc83add33525ca94eec7a8dd22c5fb1617f7a4257e9579c9e6754b37776e0877" diff --git a/internal/cmd/preprocessor/cmd/rootfile.go b/internal/cmd/preprocessor/cmd/rootfile.go index 9b6086872..1b885856a 100644 --- a/internal/cmd/preprocessor/cmd/rootfile.go +++ b/internal/cmd/preprocessor/cmd/rootfile.go @@ -382,7 +382,7 @@ func (rf *rootFile) buildMultistepScript() (*multiStepScript, error) { // TODO: work out why we are not inheriting PATH from the setpriv environment // whereas we do when we docker run from the command line. - pf("export PATH=\"/go/bin:/usr/local/go/bin:$PATH\"\n") + pf("export PATH=\"/cues/prerelease:/go/bin:/usr/local/go/bin:$PATH\"\n") // exitCodeVar is the name of the "temporary" variable used to capture // the exit code from a command. Named something suitably esoteric to diff --git a/internal/cmd/preprocessor/cmd/script_node.go b/internal/cmd/preprocessor/cmd/script_node.go index 2f6e975ce..1e7ea8678 100644 --- a/internal/cmd/preprocessor/cmd/script_node.go +++ b/internal/cmd/preprocessor/cmd/script_node.go @@ -78,6 +78,13 @@ func (s *scriptNode) validate() { // Now render each statement, creating doc comments for each as we go, and // gathering any non-doc comments as script node-level comments. + envSubstCUEVersions := func(v string) string { + return expand(v, func(vv string) (bool, string) { + version, ok := s.cueEnvVersions[vv] + return ok, version + }) + } + for _, stmt := range file.Stmts { cmdStmt := commandStmt{ stmt: stmt, @@ -130,14 +137,14 @@ func (s *scriptNode) validate() { s.errorf("%v: failed to print statement at %v: %v", s, stmt.Position, err) continue } - cmdStmt.Cmd = sb.String() + cmdStmt.Cmd = envSubstCUEVersions(sb.String()) sb.Reset() if err := s.rf.shellPrinter.Print(&sb, &doc); err != nil { s.errorf("%v: failed to print doc comment for stmt at %v: %v", s, stmt.Position, err) continue } - cmdStmt.Doc = sb.String() + cmdStmt.Doc = envSubstCUEVersions(sb.String()) // Now check if there are any known tag-based sanitiser directives // diff --git a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar index 77e70e87e..4e1de4a1e 100644 --- a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar +++ b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar @@ -28,8 +28,10 @@ exec preprocessor execute --debug=all --skipcache stderr $WORK${/}'content'${/}'dir'${/}'en.md: skipping cache for multi-step script; was a hit' -- hugo/.keep -- --- content/site.cue -- +-- site.cue -- package site + +versions: cue: someversion: "v0.7.0" -- content/dir/page.cue -- package site @@ -110,6 +112,10 @@ content: dir: page: { >nonexistent command >{{{end}}} >{{{end}}} +> +>{{{with script "en" "subst env var"}}} +>echo $CUELANG_CUE_SOMEVERSION +>{{{end}}} -- golden/content/dir/en.md.writeBack -- >--- >title: JSON Superset @@ -156,6 +162,10 @@ content: dir: page: { >nonexistent command >{{{end}}} >{{{end}}} +> +>{{{with script "en" "subst env var"}}} +>echo $CUELANG_CUE_SOMEVERSION +>{{{end}}} -- golden/hugo/content/en/dir/index.md -- --- title: JSON Superset @@ -210,6 +220,11 @@ go version go1.22.0 $ nonexistent command ``` {{< /step >}} + +```text { title="TERMINAL" codeToCopy="ZWNobyB2MC43LjAK" } +$ echo v0.7.0 +v0.7.0 +``` -- golden/content/dir/gen_cache.cue -- package site { @@ -218,12 +233,12 @@ package site page: { cache: { upload: { - "upload-some-cue": "7IDjGd2+UaDqL/CtCzWTF/n9JAghLOal/XYmGOXLTwU=" - "upload-some-json": "r7ZxaLhkjrkTxjeWFPVjLXXQEAk5pMHlkexqx+RnHKU=" - "in-subdir": "J43PmA4U4WvVNMgjwuRuxRYV01FPIWNFmKMuvNPDah4=" + "upload-some-cue": "tP3KoyKTIn79Gk8U9FATNuou3WfinCe3OvMMJIGDL+Y=" + "upload-some-json": "xZav4QaIGOVUoGgl6FLH1FyMGWS9+cOFaTTEoD5mc1U=" + "in-subdir": "WFUtw4fPeq+fBaYmRr1v6Va3OJiB3Pht2bJZgs+whQU=" } multi_step: { - "CJTPSGKT1S5VQHE852UV8UR3GU4QGPUVRF75R4F6G64J90U51250====": [{ + "J523PNTTH7RBNDJU4R39J6VCBERKCGAQASGSROP9NVDJ7EEPFC4G====": [{ doc: """ # script doc comment #scripttag @@ -270,6 +285,14 @@ package site GOARCH amd64 GOOS linux + """ + }, { + doc: "" + cmd: "echo v0.7.0" + exitCode: 0 + output: """ + v0.7.0 + """ }] } diff --git a/playground/src/config/gen_cuelang_org_go_version.ts b/playground/src/config/gen_cuelang_org_go_version.ts index a23c89b85..9c2880f55 100644 --- a/playground/src/config/gen_cuelang_org_go_version.ts +++ b/playground/src/config/gen_cuelang_org_go_version.ts @@ -1,3 +1,4 @@ // Code generated site_tool.cue; DO NOT EDIT. +// TODO: move away from defaulting to the pre-release of CUE export const CUEVersion = 'v0.8.0-alpha.1'; diff --git a/site.cue b/site.cue index 59ecd1a95..1cd86078e 100644 --- a/site.cue +++ b/site.cue @@ -11,8 +11,12 @@ import ( versions: { go: "go1.22.0" bareGoVersion: strings.TrimPrefix(go, "go") - cue: "v0.8.0-alpha.1" - testscript: "v1.11.0" + cue: { + latest: "v0.7.1" + prerelease: "v0.8.0-alpha.1" + tip: "v0.8.0-alpha.1" + } + testscript: "v1.11.0" } // _contentDefaults is a recursive template for setting defaults @@ -74,18 +78,33 @@ template: ci.#writefs & { export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ go install -trimpath github.com/rogpeppe/go-internal/cmd/testscript@\#(versions.testscript) - RUN \ - --mount=type=cache,target=/cache/gocache \ - --mount=type=cache,target=/cache/gomodcache \ - export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ - go install -trimpath cuelang.org/go/cmd/cue@\#(versions.cue) + RUN mkdir /cues + + \#(strings.Join([for name, version in versions.cue { + #""" + RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/\#(name) go install -trimpath cuelang.org/go/cmd/cue@\#(version) + """# + }], "\n\n")) FROM golang:\#(versions.bareGoVersion) RUN mkdir -p /go/bin + # TODO: move away from defaulting to the pre-release of CUE + ENV PATH="/cues/prerelease:${PATH}" + ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" - ENV CUE_VERSION="\#(versions.cue)" + \#( + strings.Join([for name, version in versions.cue { + """ + ENV CUELANG_CUE_\(strings.ToUpper(name))="\(version)" + """ + }, + ], "\n")) WORKDIR / @@ -95,7 +114,13 @@ template: ci.#writefs & { RUN chown root:root /usr/bin/entrypoint.sh COPY --from=build /go/bin/testscript /go/bin - COPY --from=build /go/bin/cue /go/bin + \#( + strings.Join([for name, version in versions.cue { + """ + COPY --from=build /cues/\(name)/cue /cues/\(name)/cue + """ + }, + ], "\n")) ENTRYPOINT ["/usr/bin/entrypoint.sh"] @@ -123,7 +148,9 @@ template: ci.#writefs & { # Contents allows for markdown, leave out the button if you don't want a button [notification] type = 'test' - content = '**Note:** documentation on this site relies on CUE \#(versions.cue)' + + # TODO: move away from defaulting to the pre-release of CUE + content = '**Note:** documentation on this site relies on CUE \#(versions.cue.prerelease)' [notification.button] link = 'https://github.com/cue-lang/cue/releases' icon = 'download' @@ -184,7 +211,8 @@ template: ci.#writefs & { Contents: #""" // \#(donotedit) - export const CUEVersion = '\#(versions.cue)'; + // TODO: move away from defaulting to the pre-release of CUE + export const CUEVersion = '\#(versions.cue.prerelease)'; """# }