From 8214884c4f7edcde4607cc434069a45d1cb87e8b Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Sat, 17 Feb 2024 06:02:20 +0000 Subject: [PATCH] preprocessor: install multiple versions of CUE in docker image WIP Preprocessor-No-Write-Cache: true Signed-off-by: Paul Jolly Change-Id: Ie8f2da1cac137ce8c390b5c6d08dcb449c4ff78a --- .../gen_cache.cue | 18 --- .../cmd/preprocessor/cmd/_docker/Dockerfile | 27 ++++- internal/cmd/preprocessor/cmd/execute.go | 17 +++ .../cmd/preprocessor/cmd/execute_context.go | 15 +++ internal/cmd/preprocessor/cmd/expand.go | 104 ++++++++++++++++++ internal/cmd/preprocessor/cmd/expand_test.go | 65 +++++++++++ .../preprocessor/cmd/gen_dockerimagetag.go | 2 +- internal/cmd/preprocessor/cmd/rootfile.go | 2 +- internal/cmd/preprocessor/cmd/script_node.go | 11 +- .../testdata/execute_multistagescript.txtar | 33 +++++- site.cue | 47 ++++++-- 11 files changed, 300 insertions(+), 41 deletions(-) delete mode 100644 content/docs/howto/use-net-ipcidr-to-validate-ip-cidr-ranges/gen_cache.cue create mode 100644 internal/cmd/preprocessor/cmd/expand.go create mode 100644 internal/cmd/preprocessor/cmd/expand_test.go diff --git a/content/docs/howto/use-net-ipcidr-to-validate-ip-cidr-ranges/gen_cache.cue b/content/docs/howto/use-net-ipcidr-to-validate-ip-cidr-ranges/gen_cache.cue deleted file mode 100644 index 6f579d3b0..000000000 --- a/content/docs/howto/use-net-ipcidr-to-validate-ip-cidr-ranges/gen_cache.cue +++ /dev/null @@ -1,18 +0,0 @@ -package site -{ - content: { - docs: { - howto: { - "use-net-ipcidr-to-validate-ip-cidr-ranges": { - page: { - cache: { - code: { - cc: "7l2IVOflrdQAXXmvhtey8iYg5NSVqrAIICrwjj96kC0=" - } - } - } - } - } - } - } -} diff --git a/internal/cmd/preprocessor/cmd/_docker/Dockerfile b/internal/cmd/preprocessor/cmd/_docker/Dockerfile index 7189a1727..7d7132007 100644 --- a/internal/cmd/preprocessor/cmd/_docker/Dockerfile +++ b/internal/cmd/preprocessor/cmd/_docker/Dockerfile @@ -15,18 +15,37 @@ RUN \ export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ go install -trimpath github.com/rogpeppe/go-internal/cmd/testscript@v1.11.0 +RUN mkdir /cues + +RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/latest go install -trimpath cuelang.org/go/cmd/cue@v0.7.1 + RUN \ --mount=type=cache,target=/cache/gocache \ --mount=type=cache,target=/cache/gomodcache \ export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ - go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 + GOBIN=/cues/prerelease go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 + +RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/tip go install -trimpath cuelang.org/go/cmd/cue@v0.8.0-alpha.1 FROM golang:1.22.0 RUN mkdir -p /go/bin +# TODO: make this more principled with respect to cue.versions +ENV PATH="/cues/prerelease:${PATH}" + ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" -ENV CUE_VERSION="v0.8.0-alpha.1" +ENV CUELANG_CUE_LATEST="v0.7.1" +ENV CUELANG_CUE_PRERELEASE="v0.8.0-alpha.1" +ENV CUELANG_CUE_TIP="v0.8.0-alpha.1" WORKDIR / @@ -36,6 +55,8 @@ RUN chmod 755 /usr/bin/entrypoint.sh RUN chown root:root /usr/bin/entrypoint.sh COPY --from=build /go/bin/testscript /go/bin -COPY --from=build /go/bin/cue /go/bin +COPY --from=build /cues/latest/cue /cues/latest/cue +COPY --from=build /cues/prerelease/cue /cues/prerelease/cue +COPY --from=build /cues/tip/cue /cues/tip/cue ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/internal/cmd/preprocessor/cmd/execute.go b/internal/cmd/preprocessor/cmd/execute.go index b72d4ce60..8c8cc8c3d 100644 --- a/internal/cmd/preprocessor/cmd/execute.go +++ b/internal/cmd/preprocessor/cmd/execute.go @@ -137,6 +137,23 @@ type executionContext struct { // preprocessor. config cue.Value + // cueVersions is a map of name to version of CUE versions configured on the + // site. e.g. + // + // latest => v0.7.1 + // prerelease => v0.8.0-alpha.1 + cueVersions map[string]string + + // cueEnvVersions is a map of env var name to CUE versions, derived from + // cueVersions. For example the entry: + // + // latest => v0.7.0 + // + // in cueVersions exists in this map as: + // + // CUELANG_CUE_LATEST => v0.7.0 + cueEnvVersions map[string]string + // selfHash is the hash of the preprocessor itself that hashing calculations // should use if they need the preprocessor to affect the hash result. selfHash string diff --git a/internal/cmd/preprocessor/cmd/execute_context.go b/internal/cmd/preprocessor/cmd/execute_context.go index c2f24c929..b01f1a367 100644 --- a/internal/cmd/preprocessor/cmd/execute_context.go +++ b/internal/cmd/preprocessor/cmd/execute_context.go @@ -117,6 +117,21 @@ func (ec *executeContext) execute() error { } ec.config = v + // Load the CUE versions configured as part of the site + ec.cueVersions = make(map[string]string) + ec.cueEnvVersions = make(map[string]string) + cueVersionsPath := cue.ParsePath("versions.cue") + cueVersions := ec.config.LookupPath(cueVersionsPath) + if cueVersions.Exists() { + if err := cueVersions.Decode(&ec.cueVersions); err != nil { + return ec.errorf("%v: failed to decode %v: %v", cueVersionsPath, err) + } + for k, v := range ec.cueVersions { + key := fmt.Sprintf("CUELANG_CUE_%s", strings.ToUpper(k)) + ec.cueEnvVersions[key] = v + } + } + // Now load config per page for _, d := range ec.order { p := ec.pages[d] diff --git a/internal/cmd/preprocessor/cmd/expand.go b/internal/cmd/preprocessor/cmd/expand.go new file mode 100644 index 000000000..d49b2b67e --- /dev/null +++ b/internal/cmd/preprocessor/cmd/expand.go @@ -0,0 +1,104 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +// The code below is copied and modified from the Go source as of +// 9cc0f9cba2f2db97f5ba6c2c482bafaaa34c0381 +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// expand replaces ${var} or $var in the string based on the mapping function +// if the mapping function returns true. +func expand(s string, mapping func(string) (bool, string)) string { + var buf []byte + // ${} is all ASCII, so bytes are fine for this operation. + i := 0 + for j := 0; j < len(s); j++ { + if s[j] == '$' && j+1 < len(s) { + if buf == nil { + buf = make([]byte, 0, 2*len(s)) + } + buf = append(buf, s[i:j]...) + name, w := getShellName(s[j+1:]) + if name == "" && w > 0 { + // Encountered invalid syntax; eat the + // characters. + } else if name == "" { + // Valid syntax, but $ was not followed by a + // name. Leave the dollar character untouched. + buf = append(buf, s[j]) + } else { + mapped, repl := mapping(name) + if mapped { + buf = append(buf, repl...) + } else { + buf = append(buf, s[j:j+1+w]...) + } + } + j += w + i = j + 1 + } + } + if buf == nil { + return s + } + return string(buf) + s[i:] +} + +// isShellSpecialVar reports whether the character identifies a special +// shell variable such as $*. +func isShellSpecialVar(c uint8) bool { + switch c { + case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore. +func isAlphaNum(c uint8) bool { + return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' +} + +// getShellName returns the name that begins the string and the number of bytes +// consumed to extract it. If the name is enclosed in {}, it's part of a ${} +// expansion and two more bytes are needed than the length of the name. +func getShellName(s string) (string, int) { + switch { + case s[0] == '{': + if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' { + return s[1:2], 3 + } + // Scan to closing brace + for i := 1; i < len(s); i++ { + if s[i] == '}' { + if i == 1 { + return "", 2 // Bad syntax; eat "${}" + } + return s[1:i], i + 1 + } + } + return "", 1 // Bad syntax; eat "${" + case isShellSpecialVar(s[0]): + return s[0:1], 1 + } + // Scan alphanumerics. + var i int + for i = 0; i < len(s) && isAlphaNum(s[i]); i++ { + } + return s[:i], i +} diff --git a/internal/cmd/preprocessor/cmd/expand_test.go b/internal/cmd/preprocessor/cmd/expand_test.go new file mode 100644 index 000000000..b78c3a15e --- /dev/null +++ b/internal/cmd/preprocessor/cmd/expand_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "testing" + + "github.com/go-quicktest/qt" +) + +func TestExpand(t *testing.T) { + mapping := func(s string) (bool, string) { + switch s { + case "BANANA": + return true, "apple" + default: + return false, "" + } + } + cases := []struct { + name string + input string + want string + }{ + { + name: "entire string", + input: "$BANANA", + want: "apple", + }, + { + name: "double dollar", // nothing replace + input: "$$BANANA", + want: "$$BANANA", + }, + { + name: "braces", + input: "${BANANA}", + want: "apple", + }, + { + name: "another var", // left untouched + input: "$SOMETHING", + want: "$SOMETHING", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := expand(c.input, mapping) + qt.Assert(t, qt.Equals(got, c.want)) + }) + } +} diff --git a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go index 59a442b40..ea59cd753 100644 --- a/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go +++ b/internal/cmd/preprocessor/cmd/gen_dockerimagetag.go @@ -2,4 +2,4 @@ package cmd -const dockerImageTag = "preprocessor:6e7cb00a0a5ab45565a39b9daa398f8fa8e2faa58a7b3baae3101c59a43764b2" +const dockerImageTag = "preprocessor:fbb90662e36c9155db435f43c8739e68cd438b0d2a2c625ec197762c3ceabfa7" diff --git a/internal/cmd/preprocessor/cmd/rootfile.go b/internal/cmd/preprocessor/cmd/rootfile.go index 95255775f..55d7b888b 100644 --- a/internal/cmd/preprocessor/cmd/rootfile.go +++ b/internal/cmd/preprocessor/cmd/rootfile.go @@ -382,7 +382,7 @@ func (rf *rootFile) buildMultistepScript() (*multiStepScript, error) { // TODO: work out why we are not inheriting PATH from the setpriv environment // whereas we do when we docker run from the command line. - pf("export PATH=\"/go/bin:/usr/local/go/bin:$PATH\"\n") + pf("export PATH=\"/cues/prerelease:/go/bin:/usr/local/go/bin:$PATH\"\n") // exitCodeVar is the name of the "temporary" variable used to capture // the exit code from a command. Named something suitably esoteric to diff --git a/internal/cmd/preprocessor/cmd/script_node.go b/internal/cmd/preprocessor/cmd/script_node.go index a99640687..a6f7b2e86 100644 --- a/internal/cmd/preprocessor/cmd/script_node.go +++ b/internal/cmd/preprocessor/cmd/script_node.go @@ -78,6 +78,13 @@ func (s *scriptNode) validate() { // Now render each statement, creating doc comments for each as we go, and // gathering any non-doc comments as script node-level comments. + envSubstCUEVersions := func(v string) string { + return expand(v, func(vv string) (bool, string) { + version, ok := s.cueEnvVersions[vv] + return ok, version + }) + } + for _, stmt := range file.Stmts { cmdStmt := commandStmt{ stmt: stmt, @@ -130,14 +137,14 @@ func (s *scriptNode) validate() { s.errorf("%v: failed to print statement at %v: %v", s, stmt.Position, err) continue } - cmdStmt.Cmd = sb.String() + cmdStmt.Cmd = envSubstCUEVersions(sb.String()) sb.Reset() if err := s.rf.shellPrinter.Print(&sb, &doc); err != nil { s.errorf("%v: failed to print doc comment for stmt at %v: %v", s, stmt.Position, err) continue } - cmdStmt.Doc = sb.String() + cmdStmt.Doc = envSubstCUEVersions(sb.String()) // Now check if there are any known tag-based sanitiser directives // diff --git a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar index 77e70e87e..6db40fade 100644 --- a/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar +++ b/internal/cmd/preprocessor/cmd/testdata/execute_multistagescript.txtar @@ -28,8 +28,10 @@ exec preprocessor execute --debug=all --skipcache stderr $WORK${/}'content'${/}'dir'${/}'en.md: skipping cache for multi-step script; was a hit' -- hugo/.keep -- --- content/site.cue -- +-- site.cue -- package site + +versions: cue: someversion: "v0.7.0" -- content/dir/page.cue -- package site @@ -110,6 +112,10 @@ content: dir: page: { >nonexistent command >{{{end}}} >{{{end}}} +> +>{{{with script "en" "subst env var"}}} +>echo $CUELANG_CUE_SOMEVERSION +>{{{end}}} -- golden/content/dir/en.md.writeBack -- >--- >title: JSON Superset @@ -156,6 +162,10 @@ content: dir: page: { >nonexistent command >{{{end}}} >{{{end}}} +> +>{{{with script "en" "subst env var"}}} +>echo $CUELANG_CUE_SOMEVERSION +>{{{end}}} -- golden/hugo/content/en/dir/index.md -- --- title: JSON Superset @@ -210,6 +220,11 @@ go version go1.22.0 $ nonexistent command ``` {{< /step >}} + +```text { title="TERMINAL" codeToCopy="ZWNobyB2MC43LjAK" } +$ echo v0.7.0 +v0.7.0 +``` -- golden/content/dir/gen_cache.cue -- package site { @@ -218,12 +233,12 @@ package site page: { cache: { upload: { - "upload-some-cue": "7IDjGd2+UaDqL/CtCzWTF/n9JAghLOal/XYmGOXLTwU=" - "upload-some-json": "r7ZxaLhkjrkTxjeWFPVjLXXQEAk5pMHlkexqx+RnHKU=" - "in-subdir": "J43PmA4U4WvVNMgjwuRuxRYV01FPIWNFmKMuvNPDah4=" + "upload-some-cue": "mk/eGLXQi8f2VHFEAJ+OVwSB0C/zK9S8nvtSvKYNyVo=" + "upload-some-json": "aOFY1SpvCxmIPWnBoZDfTCvL90OvzbTTD3gqObCa75w=" + "in-subdir": "zLF5NHI4Y1eaRJ7n4t6bN//ptgOj2snRyP3DNUSUZrw=" } multi_step: { - "CJTPSGKT1S5VQHE852UV8UR3GU4QGPUVRF75R4F6G64J90U51250====": [{ + "L4H91P78HIDIAEMM7U0CA93FTU24VD5B7K7E0BRUPJPUM8AV9RV0====": [{ doc: """ # script doc comment #scripttag @@ -270,6 +285,14 @@ package site GOARCH amd64 GOOS linux + """ + }, { + doc: "" + cmd: "echo v0.7.0" + exitCode: 0 + output: """ + v0.7.0 + """ }] } diff --git a/site.cue b/site.cue index 59ecd1a95..1f370ff44 100644 --- a/site.cue +++ b/site.cue @@ -11,8 +11,12 @@ import ( versions: { go: "go1.22.0" bareGoVersion: strings.TrimPrefix(go, "go") - cue: "v0.8.0-alpha.1" - testscript: "v1.11.0" + cue: { + latest: "v0.7.1" + prerelease: "v0.8.0-alpha.1" + tip: "v0.8.0-alpha.1" + } + testscript: "v1.11.0" } // _contentDefaults is a recursive template for setting defaults @@ -74,18 +78,33 @@ template: ci.#writefs & { export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ go install -trimpath github.com/rogpeppe/go-internal/cmd/testscript@\#(versions.testscript) - RUN \ - --mount=type=cache,target=/cache/gocache \ - --mount=type=cache,target=/cache/gomodcache \ - export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ - go install -trimpath cuelang.org/go/cmd/cue@\#(versions.cue) + RUN mkdir /cues + + \#(strings.Join([for name, version in versions.cue { + #""" + RUN \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \ + GOBIN=/cues/\#(name) go install -trimpath cuelang.org/go/cmd/cue@\#(version) + """# + }], "\n\n")) FROM golang:\#(versions.bareGoVersion) RUN mkdir -p /go/bin + # TODO: make this more principled with respect to cue.versions + ENV PATH="/cues/prerelease:${PATH}" + ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" - ENV CUE_VERSION="\#(versions.cue)" + \#( + strings.Join([for name, version in versions.cue { + """ + ENV CUELANG_CUE_\(strings.ToUpper(name))="\(version)" + """ + }, + ], "\n")) WORKDIR / @@ -95,7 +114,13 @@ template: ci.#writefs & { RUN chown root:root /usr/bin/entrypoint.sh COPY --from=build /go/bin/testscript /go/bin - COPY --from=build /go/bin/cue /go/bin + \#( + strings.Join([for name, version in versions.cue { + """ + COPY --from=build /cues/\(name)/cue /cues/\(name)/cue + """ + }, + ], "\n")) ENTRYPOINT ["/usr/bin/entrypoint.sh"] @@ -123,7 +148,7 @@ template: ci.#writefs & { # Contents allows for markdown, leave out the button if you don't want a button [notification] type = 'test' - content = '**Note:** documentation on this site relies on CUE \#(versions.cue)' + content = '**Note:** documentation on this site relies on CUE \#(versions.cue.prerelease)' [notification.button] link = 'https://github.com/cue-lang/cue/releases' icon = 'download' @@ -184,7 +209,7 @@ template: ci.#writefs & { Contents: #""" // \#(donotedit) - export const CUEVersion = '\#(versions.cue)'; + export const CUEVersion = '\#(versions.cue.prerelease)'; """# }