preprocessor: install multiple versions of CUE in docker image

WIP

Preprocessor-No-Write-Cache: true
Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Ie8f2da1cac137ce8c390b5c6d08dcb449c4ff78a
Dispatch-Trailer: {"type":"trybot","CL":1177003,"patchset":3,"ref":"refs/changes/03/1177003/3","targetBranch":"alpha"}

internal/cmd/preprocessor/cmd/_docker/Dockerfile

@@ -15,18 +15,37 @@ RUN \
   export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \
   go install -trimpath github.com/rogpeppe/go-internal/cmd/[email protected]
 
+RUN mkdir /cues
+
+RUN \
+  --mount=type=cache,target=/cache/gocache \
+  --mount=type=cache,target=/cache/gomodcache \
+  export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \
+  GOBIN=/cues/latest go install -trimpath cuelang.org/go/cmd/[email protected]
+
 RUN \
   --mount=type=cache,target=/cache/gocache \
   --mount=type=cache,target=/cache/gomodcache \
   export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \
-  go install -trimpath cuelang.org/go/cmd/[email protected]
+  GOBIN=/cues/prerelease go install -trimpath cuelang.org/go/cmd/[email protected]
+
+RUN \
+  --mount=type=cache,target=/cache/gocache \
+  --mount=type=cache,target=/cache/gomodcache \
+  export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache && \
+  GOBIN=/cues/tip go install -trimpath cuelang.org/go/cmd/[email protected]
 
 FROM golang:1.22.0
 
 RUN mkdir -p /go/bin
 
+# TODO: make this more principled with respect to cue.versions
+ENV PATH="/cues/prerelease:${PATH}"
+
 ENV PATH="/go/bin:/usr/local/go/bin:${PATH}"
-ENV CUE_VERSION="v0.8.0-alpha.1"
+ENV CUELANG_CUE_LATEST="v0.7.1"
+ENV CUELANG_CUE_PRERELEASE="v0.8.0-alpha.1"
+ENV CUELANG_CUE_TIP="v0.8.0-alpha.1"
 
 WORKDIR /
 
@@ -36,6 +55,8 @@ RUN chmod 755 /usr/bin/entrypoint.sh
 RUN chown root:root /usr/bin/entrypoint.sh
 
 COPY --from=build /go/bin/testscript /go/bin
-COPY --from=build /go/bin/cue /go/bin
+COPY --from=build /cues/latest/cue /cues/latest/cue
+COPY --from=build /cues/prerelease/cue /cues/prerelease/cue
+COPY --from=build /cues/tip/cue /cues/tip/cue
 
 ENTRYPOINT ["/usr/bin/entrypoint.sh"]

internal/cmd/preprocessor/cmd/execute.go

@@ -137,6 +137,23 @@ type executionContext struct {
 	// preprocessor.
 	config cue.Value
 
+	// cueVersions is a map of name to version of CUE versions configured on the
+	// site. e.g.
+	//
+	// latest => v0.7.1
+	// prerelease => v0.8.0-alpha.1
+	cueVersions map[string]string
+
+	// cueEnvVersions is a map of env var name to CUE versions, derived from
+	// cueVersions. For example the entry:
+	//
+	// latest => v0.7.0
+	//
+	// in cueVersions exists in this map as:
+	//
+	// CUELANG_CUE_LATEST => v0.7.0
+	cueEnvVersions map[string]string
+
 	// selfHash is the hash of the preprocessor itself that hashing calculations
 	// should use if they need the preprocessor to affect the hash result.
 	selfHash string

internal/cmd/preprocessor/cmd/execute_context.go

@@ -117,6 +117,21 @@ func (ec *executeContext) execute() error {
 	}
 	ec.config = v
 
+	// Load the CUE versions configured as part of the site
+	ec.cueVersions = make(map[string]string)
+	ec.cueEnvVersions = make(map[string]string)
+	cueVersionsPath := cue.ParsePath("versions.cue")
+	cueVersions := ec.config.LookupPath(cueVersionsPath)
+	if cueVersions.Exists() {
+		if err := cueVersions.Decode(&ec.cueVersions); err != nil {
+			return ec.errorf("%v: failed to decode %v: %v", cueVersionsPath, err)
+		}
+		for k, v := range ec.cueVersions {
+			key := fmt.Sprintf("CUELANG_CUE_%s", strings.ToUpper(k))
+			ec.cueEnvVersions[key] = v
+		}
+	}
+
 	// Now load config per page
 	for _, d := range ec.order {
 		p := ec.pages[d]

internal/cmd/preprocessor/cmd/expand.go

@@ -0,0 +1,104 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+// The code below is copied and modified from the Go source as of
+// 9cc0f9cba2f2db97f5ba6c2c482bafaaa34c0381
+//
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// expand replaces ${var} or $var in the string based on the mapping function
+// if the mapping function returns true.
+func expand(s string, mapping func(string) (bool, string)) string {
+	var buf []byte
+	// ${} is all ASCII, so bytes are fine for this operation.
+	i := 0
+	for j := 0; j < len(s); j++ {
+		if s[j] == '$' && j+1 < len(s) {
+			if buf == nil {
+				buf = make([]byte, 0, 2*len(s))
+			}
+			buf = append(buf, s[i:j]...)
+			name, w := getShellName(s[j+1:])
+			if name == "" && w > 0 {
+				// Encountered invalid syntax; eat the
+				// characters.
+			} else if name == "" {
+				// Valid syntax, but $ was not followed by a
+				// name. Leave the dollar character untouched.
+				buf = append(buf, s[j])
+			} else {
+				mapped, repl := mapping(name)
+				if mapped {
+					buf = append(buf, repl...)
+				} else {
+					buf = append(buf, s[j:j+1+w]...)
+				}
+			}
+			j += w
+			i = j + 1
+		}
+	}
+	if buf == nil {
+		return s
+	}
+	return string(buf) + s[i:]
+}
+
+// isShellSpecialVar reports whether the character identifies a special
+// shell variable such as $*.
+func isShellSpecialVar(c uint8) bool {
+	switch c {
+	case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		return true
+	}
+	return false
+}
+
+// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore.
+func isAlphaNum(c uint8) bool {
+	return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z'
+}
+
+// getShellName returns the name that begins the string and the number of bytes
+// consumed to extract it. If the name is enclosed in {}, it's part of a ${}
+// expansion and two more bytes are needed than the length of the name.
+func getShellName(s string) (string, int) {
+	switch {
+	case s[0] == '{':
+		if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' {
+			return s[1:2], 3
+		}
+		// Scan to closing brace
+		for i := 1; i < len(s); i++ {
+			if s[i] == '}' {
+				if i == 1 {
+					return "", 2 // Bad syntax; eat "${}"
+				}
+				return s[1:i], i + 1
+			}
+		}
+		return "", 1 // Bad syntax; eat "${"
+	case isShellSpecialVar(s[0]):
+		return s[0:1], 1
+	}
+	// Scan alphanumerics.
+	var i int
+	for i = 0; i < len(s) && isAlphaNum(s[i]); i++ {
+	}
+	return s[:i], i
+}

internal/cmd/preprocessor/cmd/expand_test.go

@@ -0,0 +1,65 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/go-quicktest/qt"
+)
+
+func TestExpand(t *testing.T) {
+	mapping := func(s string) (bool, string) {
+		switch s {
+		case "BANANA":
+			return true, "apple"
+		default:
+			return false, ""
+		}
+	}
+	cases := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			name:  "entire string",
+			input: "$BANANA",
+			want:  "apple",
+		},
+		{
+			name:  "double dollar", // nothing replace
+			input: "$$BANANA",
+			want:  "$$BANANA",
+		},
+		{
+			name:  "braces",
+			input: "${BANANA}",
+			want:  "apple",
+		},
+		{
+			name:  "another var", // left untouched
+			input: "$SOMETHING",
+			want:  "$SOMETHING",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := expand(c.input, mapping)
+			qt.Assert(t, qt.Equals(got, c.want))
+		})
+	}
+}

internal/cmd/preprocessor/cmd/rootfile.go

@@ -382,7 +382,7 @@ func (rf *rootFile) buildMultistepScript() (*multiStepScript, error) {
 
 	// TODO: work out why we are not inheriting PATH from the setpriv environment
 	// whereas we do when we docker run from the command line.
-	pf("export PATH=\"/go/bin:/usr/local/go/bin:$PATH\"\n")
+	pf("export PATH=\"/cues/prerelease:/go/bin:/usr/local/go/bin:$PATH\"\n")
 
 	// exitCodeVar is the name of the "temporary" variable used to capture
 	// the exit code from a command. Named something suitably esoteric to

internal/cmd/preprocessor/cmd/script_node.go

@@ -78,6 +78,13 @@ func (s *scriptNode) validate() {
 	// Now render each statement, creating doc comments for each as we go, and
 	// gathering any non-doc comments as script node-level comments.
 
+	envSubstCUEVersions := func(v string) string {
+		return expand(v, func(vv string) (bool, string) {
+			version, ok := s.cueEnvVersions[vv]
+			return ok, version
+		})
+	}
+
 	for _, stmt := range file.Stmts {
 		cmdStmt := commandStmt{
 			stmt: stmt,
@@ -130,14 +137,14 @@ func (s *scriptNode) validate() {
 			s.errorf("%v: failed to print statement at %v: %v", s, stmt.Position, err)
 			continue
 		}
-		cmdStmt.Cmd = sb.String()
+		cmdStmt.Cmd = envSubstCUEVersions(sb.String())
 
 		sb.Reset()
 		if err := s.rf.shellPrinter.Print(&sb, &doc); err != nil {
 			s.errorf("%v: failed to print doc comment for stmt at %v: %v", s, stmt.Position, err)
 			continue
 		}
-		cmdStmt.Doc = sb.String()
+		cmdStmt.Doc = envSubstCUEVersions(sb.String())
 
 		// Now check if there are any known tag-based sanitiser directives
 		//