playground: fix usage of site playground CUE version

In CL 1200796, we sort to ensure that
_scripts/revendorToolsInternal.bash was run at go:generate time.

However, what we missed in that CL is that the go:generate directive is
in main.go, which is build constraint limited to wasm/js. Hence, when
running 'go generate' (which is a build command and hence limited to
considering packages and package files that match the GOOS and GOARCH of
the command environment) we end up "missing" the go:generate directive
in main.go.

This didn't get spotted at the time, because as of that CL the running
of revendorToolsInternal.bash would have been a no-op. The lack of
synchronisation that had to be manually adjusted for in later CLs was
also missed.

Fix that by moving both go:generate directives to a build constraint
unguarded file gen.go, which is something of a regular pattern in the
CUE project.

This CL then also contains the relevant diffs that result from
revendorToolsInternal.bash actually being run.

Signed-off-by: Paul Jolly <[email protected]>
Change-Id: Iebaf41c526adf9f7de35370e95fe7be18f04345c
Dispatch-Trailer: {"type":"trybot","CL":1204975,"patchset":2,"ref":"refs/changes/75/1204975/2","targetBranch":"master"}

playground/_scripts/revendorToolsInternal.bash

@@ -9,7 +9,10 @@ cd "$rootDir/playground"
 
 playVersion="$(../_scripts/playgroundCUEVersion.bash)"
 go mod edit -require cuelang.org/go@$playVersion
-go mod tidy
+
+# Note we cannot tidy here because the playground code might not be in a valid state.
+# go mod tidy
+
 go mod download
 path="cuelang.org/go"
 version=$(go list -m -f={{.Version}} $path)
@@ -26,6 +29,8 @@ fi
 unzip -q $modCache/cache/download/$path/@v/$version.zip
 popd > /dev/null
 
+# Ensure the target of the rsync exists
+mkdir -p ./internal/cuelang_org_go_internal
 
 # TODO this is very fragile - we need to move away from this to a proper
 # dependency-based approach to minimal copy
@@ -47,3 +52,6 @@ find ./internal/cuelang_org_go_internal/ -name "*_test.go" -exec rm {} +
 
 # Retain a copy of the license
 cp $td/$path@$version/LICENSE ./internal/cuelang_org_go_internal
+
+# Now do a go mod tidy - this should now work.
+go mod tidy

playground/gen.go

@@ -0,0 +1,18 @@
+// Copyright 2024 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+//go:generate ./_scripts/cpWasmExec.bash
+//go:generate ./_scripts/revendorToolsInternal.bash

playground/impl.go

@@ -30,8 +30,6 @@ import (
 	"github.com/cue-lang/cuelang.org/playground/internal/cuelang_org_go_internal/filetypes"
 )
 
-//go:generate ./_scripts/cpWasmExec.bash
-
 type function string
 
 const (

playground/internal/cuelang_org_go_internal/astinternal/debugstr.go → playground/internal/cuelang_org_go_internal/astinternal/debug.go

@@ -16,6 +16,8 @@ package astinternal
 
 import (
 	"fmt"
+	gotoken "go/token"
+	"reflect"
 	"strconv"
 	"strings"
 
@@ -24,6 +26,193 @@ import (
 	"github.com/cue-lang/cuelang.org/playground/internal/cuelang_org_go_internal"
 )
 
+// AppendDebug writes a multi-line Go-like representation of a syntax tree node,
+// including node position information and any relevant Go types.
+func AppendDebug(dst []byte, node ast.Node, config DebugConfig) []byte {
+	d := &debugPrinter{
+		cfg: config,
+		buf: dst,
+	}
+	if d.value(reflect.ValueOf(node), nil) {
+		d.newline()
+	}
+	return d.buf
+}
+
+// DebugConfig configures the behavior of [AppendDebug].
+type DebugConfig struct {
+	// Filter is called before each value in a syntax tree.
+	// Values for which the function returns false are omitted.
+	Filter func(reflect.Value) bool
+
+	// OmitEmpty causes empty strings, empty structs, empty lists,
+	// nil pointers, invalid positions, and missing tokens to be omitted.
+	OmitEmpty bool
+}
+
+type debugPrinter struct {
+	buf   []byte
+	cfg   DebugConfig
+	level int
+}
+
+// value produces the given value, omitting type information if
+// its type is the same as implied type. It reports whether
+// anything was produced.
+func (d *debugPrinter) value(v reflect.Value, impliedType reflect.Type) bool {
+	start := d.pos()
+	d.value0(v, impliedType)
+	return d.pos() > start
+}
+
+func (d *debugPrinter) value0(v reflect.Value, impliedType reflect.Type) {
+	if d.cfg.Filter != nil && !d.cfg.Filter(v) {
+		return
+	}
+	// Skip over interfaces and pointers, stopping early if nil.
+	concreteType := v.Type()
+	for {
+		k := v.Kind()
+		if k != reflect.Interface && k != reflect.Pointer {
+			break
+		}
+		if v.IsNil() {
+			if !d.cfg.OmitEmpty {
+				d.printf("nil")
+			}
+			return
+		}
+		v = v.Elem()
+		if k == reflect.Interface {
+			// For example, *ast.Ident can be the concrete type behind an ast.Expr.
+			concreteType = v.Type()
+		}
+	}
+
+	if d.cfg.OmitEmpty && v.IsZero() {
+		return
+	}
+
+	t := v.Type()
+	switch v := v.Interface().(type) {
+	// Simple types which can stringify themselves.
+	case token.Pos:
+		d.printf("%s(%q", t, v)
+		// Show relative positions too, if there are any, as they affect formatting.
+		if v.HasRelPos() {
+			d.printf(", %v", v.RelPos())
+		}
+		d.printf(")")
+		return
+	case token.Token:
+		d.printf("%s(%q)", t, v)
+		return
+	}
+
+	switch t.Kind() {
+	default:
+		// We assume all other kinds are basic in practice, like string or bool.
+		if t.PkgPath() != "" {
+			// Mention defined and non-predeclared types, for clarity.
+			d.printf("%s(%#v)", t, v)
+		} else {
+			d.printf("%#v", v)
+		}
+
+	case reflect.Slice, reflect.Struct:
+		valueStart := d.pos()
+		// We print the concrete type when it differs from an implied type.
+		if concreteType != impliedType {
+			d.printf("%s", concreteType)
+		}
+		d.printf("{")
+		d.level++
+		var anyElems bool
+		if t.Kind() == reflect.Slice {
+			anyElems = d.sliceElems(v, t.Elem())
+		} else {
+			anyElems = d.structFields(v)
+		}
+		d.level--
+		if !anyElems && d.cfg.OmitEmpty {
+			d.truncate(valueStart)
+		} else {
+			if anyElems {
+				d.newline()
+			}
+			d.printf("}")
+		}
+	}
+}
+
+func (d *debugPrinter) sliceElems(v reflect.Value, elemType reflect.Type) (anyElems bool) {
+	for i := 0; i < v.Len(); i++ {
+		ev := v.Index(i)
+		elemStart := d.pos()
+		d.newline()
+		// Note: a slice literal implies the type of its elements
+		// so we can avoid mentioning the type
+		// of each element if it matches.
+		if d.value(ev, elemType) {
+			anyElems = true
+		} else {
+			d.truncate(elemStart)
+		}
+	}
+	return anyElems
+}
+
+func (d *debugPrinter) structFields(v reflect.Value) (anyElems bool) {
+	t := v.Type()
+	for i := 0; i < v.NumField(); i++ {
+		f := t.Field(i)
+		if !gotoken.IsExported(f.Name) {
+			continue
+		}
+		switch f.Name {
+		// These fields are cyclic, and they don't represent the syntax anyway.
+		case "Scope", "Node", "Unresolved":
+			continue
+		}
+		elemStart := d.pos()
+		d.newline()
+		d.printf("%s: ", f.Name)
+		if d.value(v.Field(i), nil) {
+			anyElems = true
+		} else {
+			d.truncate(elemStart)
+		}
+	}
+	val := v.Addr().Interface()
+	if val, ok := val.(ast.Node); ok {
+		// Comments attached to a node aren't a regular field, but are still useful.
+		// The majority of nodes won't have comments, so skip them when empty.
+		if comments := ast.Comments(val); len(comments) > 0 {
+			anyElems = true
+			d.newline()
+			d.printf("Comments: ")
+			d.value(reflect.ValueOf(comments), nil)
+		}
+	}
+	return anyElems
+}
+
+func (d *debugPrinter) printf(format string, args ...any) {
+	d.buf = fmt.Appendf(d.buf, format, args...)
+}
+
+func (d *debugPrinter) newline() {
+	d.buf = fmt.Appendf(d.buf, "\n%s", strings.Repeat("\t", d.level))
+}
+
+func (d *debugPrinter) pos() int {
+	return len(d.buf)
+}
+
+func (d *debugPrinter) truncate(pos int) {
+	d.buf = d.buf[:pos]
+}
+
 func DebugStr(x interface{}) (out string) {
 	if n, ok := x.(ast.Node); ok {
 		comments := ""

playground/internal/cuelang_org_go_internal/cueexperiment/exp.go

@@ -11,18 +11,30 @@ import (
 // When adding, deleting, or modifying entries below,
 // update cmd/cue/cmd/help.go as well for `cue help environment`.
 var Flags struct {
-	Modules bool `envflag:"default:true"`
+	Modules bool `envflag:"deprecated,default:true"`
 
 	// YAMLV3Decoder swaps the old internal/third_party/yaml decoder with the new
 	// decoder implemented in internal/encoding/yaml on top of yaml.v3.
-	YAMLV3Decoder bool `envflag:"default:true"`
+	// We keep it around for v0.11 for the sake of not breaking users
+	// with CUE_EXPERIMENT=yamlv3decoder=1 who must still suppport older CUE versions,
+	// but currently the feature is always enabled.
+	// TODO(mvdan): remove for v0.12.
+	YAMLV3Decoder bool `envflag:"deprecated,default:true"`
 
 	// EvalV3 enables the new evaluator. The new evaluator addresses various
 	// performance concerns.
 	EvalV3 bool
 
-	// Embed enabled file embedding.
+	// Embed enables file embedding.
 	Embed bool
+
+	// DecodeInt64 changes [cuelang.org/go/cue.Value.Decode] to choose
+	// `int64` rather than `int` as the default type for CUE integer values
+	// to ensure consistency with 32-bit platforms.
+	DecodeInt64 bool
+
+	// Enable topological sorting of struct fields
+	TopoSort bool
 }
 
 // Init initializes Flags. Note: this isn't named "init" because we

playground/internal/cuelang_org_go_internal/encoding/encoder.go

@@ -33,9 +33,9 @@ import (
 	"cuelang.org/go/encoding/protobuf/jsonpb"
 	"cuelang.org/go/encoding/protobuf/textproto"
 	"cuelang.org/go/encoding/toml"
+	"cuelang.org/go/encoding/yaml"
 	"github.com/cue-lang/cuelang.org/playground/internal/cuelang_org_go_internal"
 	"github.com/cue-lang/cuelang.org/playground/internal/cuelang_org_go_internal/filetypes"
-	"cuelang.org/go/pkg/encoding/yaml"
 )
 
 // An Encoder converts CUE to various file formats, including CUE itself.
@@ -69,10 +69,7 @@ func (e Encoder) Close() error {
 
 // NewEncoder writes content to the file with the given specification.
 func NewEncoder(ctx *cue.Context, f *build.File, cfg *Config) (*Encoder, error) {
-	w, close, err := writer(f, cfg)
-	if err != nil {
-		return nil, err
-	}
+	w, close := writer(f, cfg)
 	e := &Encoder{
 		ctx:   ctx,
 		cfg:   cfg,
@@ -182,17 +179,18 @@ func NewEncoder(ctx *cue.Context, f *build.File, cfg *Config) (*Encoder, error)
 	case build.YAML:
 		e.concrete = true
 		streamed := false
+		// TODO(mvdan): use a NewEncoder API like in TOML below.
 		e.encValue = func(v cue.Value) error {
 			if streamed {
 				fmt.Fprintln(w, "---")
 			}
 			streamed = true
 
-			str, err := yaml.Marshal(v)
+			b, err := yaml.Encode(v)
 			if err != nil {
 				return err
 			}
-			_, err = fmt.Fprint(w, str)
+			_, err = w.Write(b)
 			return err
 		}
 
@@ -289,16 +287,16 @@ func (e *Encoder) encodeFile(f *ast.File, interpret func(cue.Value) (*ast.File,
 	return e.encValue(v)
 }
 
-func writer(f *build.File, cfg *Config) (_ io.Writer, close func() error, err error) {
+func writer(f *build.File, cfg *Config) (_ io.Writer, close func() error) {
 	if cfg.Out != nil {
-		return cfg.Out, nil, nil
+		return cfg.Out, nil
 	}
 	path := f.Filename
 	if path == "-" {
 		if cfg.Stdout == nil {
-			return os.Stdout, nil, nil
+			return os.Stdout, nil
 		}
-		return cfg.Stdout, nil, nil
+		return cfg.Stdout, nil
 	}
 	// Delay opening the file until we can write it to completion.
 	// This prevents clobbering the file in case of a crash.
@@ -309,7 +307,7 @@ func writer(f *build.File, cfg *Config) (_ io.Writer, close func() error, err er
 			// Swap O_EXCL for O_TRUNC to allow replacing an entire existing file.
 			mode = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
 		}
-		f, err := os.OpenFile(path, mode, 0o644)
+		f, err := os.OpenFile(path, mode, 0o666)
 		if err != nil {
 			if errors.Is(err, fs.ErrExist) {
 				return errors.Wrapf(fs.ErrExist, token.NoPos, "error writing %q", path)
@@ -322,5 +320,5 @@ func writer(f *build.File, cfg *Config) (_ io.Writer, close func() error, err er
 		}
 		return err
 	}
-	return b, fn, nil
+	return b, fn
 }