Store YAML source info in ordered.Map

interpolate.go

@@ -3,6 +3,7 @@ package pipeline
 import (
 	"github.com/buildkite/go-pipeline/ordered"
 	"github.com/buildkite/interpolate"
+	"gopkg.in/yaml.v3"
 )
 
 // This file contains helpers for recursively interpolating all the strings in
@@ -151,7 +152,7 @@ func interpolateMap[K comparable, V any, M ~map[K]V](tf stringTransformer, m M)
 // interpolateOrderedMap applies interpolateAny over any type of ordered.Map.
 // The map is altered in-place.
 func interpolateOrderedMap[K comparable, V any](tf stringTransformer, m *ordered.Map[K, V]) error {
-	return m.Range(func(k K, v V) error {
+	return m.Range(func(k K, v V, src *yaml.Node) error {
 		// We interpolate both keys and values.
 		intk, err := interpolateAny(tf, k)
 		if err != nil {
@@ -162,7 +163,7 @@ func interpolateOrderedMap[K comparable, V any](tf stringTransformer, m *ordered
 			return err
 		}
 
-		m.Replace(k, intk, intv)
+		m.Replace(k, intk, intv, src)
 		return nil
 	})
 }

ordered/map.go

@@ -40,11 +40,11 @@ func NewMap[K comparable, V any](cap int) *Map[K, V] {
 	}
 }
 
-// MapFromItems creates an Map with some items.
+// MapFromItems creates an Map with some items. Sources are set to nil.
 func MapFromItems[K comparable, V any](ps ...Tuple[K, V]) *Map[K, V] {
 	m := NewMap[K, V](len(ps))
 	for _, p := range ps {
-		m.Set(p.Key, p.Value)
+		m.Set(p.Key, p.Value, nil)
 	}
 	return m
 }
@@ -87,7 +87,7 @@ func (m *Map[K, V]) Contains(k K) bool {
 
 // Set sets the value for the given key. If the key exists, it remains in its
 // existing spot, otherwise it is added to the end of the map.
-func (m *Map[K, V]) Set(k K, v V) {
+func (m *Map[K, V]) Set(k K, v V, source *yaml.Node) {
 	// Suppose someone makes Map with new(Map). The one thing we need to not be
 	// nil will be nil.
 	if m.index == nil {
@@ -103,8 +103,9 @@ func (m *Map[K, V]) Set(k K, v V) {
 	// Append new item.
 	m.index[k] = len(m.items)
 	m.items = append(m.items, Tuple[K, V]{
-		Key:   k,
-		Value: v,
+		Key:    k,
+		Value:  v,
+		Source: source,
 	})
 }
 
@@ -114,7 +115,7 @@ func (m *Map[K, V]) Set(k K, v V) {
 // then it is deleted.
 // This provides a way to change a single key in-place (easier than deleting the
 // old key and all later keys, adding the new key, then restoring the rest).
-func (m *Map[K, V]) Replace(old, new K, v V) {
+func (m *Map[K, V]) Replace(old, new K, v V, src *yaml.Node) {
 	// Suppose someone makes Map with new(Map). The one thing we need to not be
 	// nil will be nil.
 	if m.index == nil {
@@ -146,8 +147,9 @@ func (m *Map[K, V]) Replace(old, new K, v V) {
 
 	// Put the item into m.items at idx.
 	m.items[idx] = Tuple[K, V]{
-		Key:   new,
-		Value: v,
+		Key:    new,
+		Value:  v,
+		Source: src,
 	}
 }
 
@@ -177,7 +179,7 @@ func (m *Map[K, V]) ToMap() map[K]V {
 		return nil
 	}
 	um := make(map[K]V, len(m.index))
-	m.Range(func(k K, v V) error {
+	m.Range(func(k K, v V, _ *yaml.Node) error {
 		um[k] = v
 		return nil
 	})
@@ -192,7 +194,7 @@ func ToMapRecursive(src any) any {
 	switch tsrc := src.(type) {
 	case *Map[string, any]:
 		um := make(map[string]any, len(tsrc.index))
-		tsrc.Range(func(k string, v any) error {
+		tsrc.Range(func(k string, v any, _ *yaml.Node) error {
 			um[k] = ToMapRecursive(v)
 			return nil
 		})
@@ -264,15 +266,15 @@ func (m *Map[K, V]) compact() {
 
 // Range ranges over the map (in order). If f returns an error, it stops ranging
 // and returns that error.
-func (m *Map[K, V]) Range(f func(k K, v V) error) error {
+func (m *Map[K, V]) Range(f func(k K, v V, s *yaml.Node) error) error {
 	if m.IsZero() {
 		return nil
 	}
 	for _, p := range m.items {
 		if p.deleted {
 			continue
 		}
-		if err := f(p.Key, p.Value); err != nil {
+		if err := f(p.Key, p.Value, p.Source); err != nil {
 			return err
 		}
 	}
@@ -286,7 +288,7 @@ func (m *Map[K, V]) MarshalJSON() ([]byte, error) {
 	var b bytes.Buffer
 	b.WriteRune('{')
 	first := true
-	err := m.Range(func(k K, v V) error {
+	err := m.Range(func(k K, v V, _ *yaml.Node) error {
 		if !first {
 			// Separating comma.
 			b.WriteRune(',')
@@ -319,7 +321,7 @@ func (m *Map[K, V]) MarshalYAML() (any, error) {
 		Kind: yaml.MappingNode,
 		Tag:  "!!map",
 	}
-	err := m.Range(func(k K, v V) error {
+	err := m.Range(func(k K, v V, _ *yaml.Node) error {
 		nk, nv := new(yaml.Node), new(yaml.Node)
 		if err := nk.Encode(k); err != nil {
 			return err
@@ -356,7 +358,7 @@ func (m *Map[K, V]) UnmarshalYAML(n *yaml.Node) error {
 	}
 
 	if n.Kind != yaml.MappingNode {
-		return fmt.Errorf("line %d, col %d: wrong kind (got %x, want %x)", n.Line, n.Column, n.Kind, yaml.MappingNode)
+		return fmt.Errorf("%swrong kind (got %x, want %x)", SourcePrefix(n), n.Kind, yaml.MappingNode)
 	}
 
 	switch tm := any(m).(type) {
@@ -372,7 +374,7 @@ func (m *Map[K, V]) UnmarshalYAML(n *yaml.Node) error {
 	case *Map[string, *yaml.Node]:
 		// Load into the map without any value decoding.
 		return rangeYAMLMap(n, func(key string, val *yaml.Node) error {
-			tm.Set(key, val)
+			tm.Set(key, val, val)
 			return nil
 		})
 
@@ -390,7 +392,7 @@ func (m *Map[K, V]) UnmarshalYAML(n *yaml.Node) error {
 					return err
 				}
 			}
-			om.Set(key, v)
+			om.Set(key, v, val)
 			return nil
 		})
 	}
@@ -401,22 +403,22 @@ func (m *Map[K, V]) UnmarshalYAML(n *yaml.Node) error {
 // assertable to V.
 func AssertValues[V any](m *MapSA) (*Map[string, V], error) {
 	msv := NewMap[string, V](m.Len())
-	return msv, m.Range(func(k string, v any) error {
+	return msv, m.Range(func(k string, v any, src *yaml.Node) error {
 		t, ok := v.(V)
 		if !ok {
-			return fmt.Errorf("value for key %q (type %T) is not assertable to %T", k, v, t)
+			return fmt.Errorf("%svalue for key %q (type %T) is not assertable to %T", SourcePrefix(src), k, v, t)
 		}
-		msv.Set(k, t)
+		msv.Set(k, t, src)
 		return nil
 	})
 }
 
 // TransformValues converts a map with V1 values into a map with V2 values by
-// running each value through a function.
+// running each value through a function. Sources are preserved.
 func TransformValues[K comparable, V1, V2 any](m *Map[K, V1], f func(V1) V2) *Map[K, V2] {
 	m2 := NewMap[K, V2](m.Len())
-	m.Range(func(k K, v V1) error {
-		m2.Set(k, f(v))
+	m.Range(func(k K, v V1, s *yaml.Node) error {
+		m2.Set(k, f(v), s)
 		return nil
 	})
 	return m2

ordered/map_test.go

@@ -138,7 +138,7 @@ func TestMapSet(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			t.Parallel()
 
-			test.input.Set(test.key, test.value)
+			test.input.Set(test.key, test.value, nil)
 			if diff := cmp.Diff(test.input, test.want, cmp.Comparer(EqualSS)); diff != "" {
 				t.Errorf("after Set(%q, %q), map diff (-got +want):\n%s", test.key, test.value, diff)
 			}
@@ -219,7 +219,7 @@ func TestMapReplace(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			t.Parallel()
 
-			test.input.Replace(test.oldkey, test.newkey, test.value)
+			test.input.Replace(test.oldkey, test.newkey, test.value, nil)
 			if diff := cmp.Diff(test.input, test.want, cmp.Comparer(EqualSS)); diff != "" {
 				t.Errorf("after Replace(%q, %q, %q), map diff (-got +want):\n%s", test.oldkey, test.newkey, test.value, diff)
 			}

ordered/tuple.go

@@ -1,10 +1,17 @@
 package ordered
 
+import (
+	"gopkg.in/yaml.v3"
+)
+
 // Tuple is used for storing values in Map.
 type Tuple[K comparable, V any] struct {
 	Key   K
 	Value V
 
+	// Source is where place where the value came from.
+	Source *yaml.Node
+
 	deleted bool
 }
 

ordered/unmarshal.go

@@ -304,13 +304,13 @@ func (m *Map[K, V]) decodeInto(target any) error {
 
 		valueType := mapType.Elem()
 		var warns []error
-		if err := tm.Range(func(k string, v any) error {
+		if err := tm.Range(func(k string, v any, src *yaml.Node) error {
 			nv := reflect.New(valueType)
 			err := Unmarshal(v, nv.Interface())
 			if w := warning.As(err); w != nil {
-				warns = append(warns, w.Wrapf("while unmarshaling value for key %q", k))
+				warns = append(warns, w.Wrapf("%swhile unmarshaling value for key %q", SourcePrefix(src), k))
 			} else if err != nil {
-				return fmt.Errorf("unmarshaling value for key %q: %w", k, err)
+				return fmt.Errorf("%sunmarshaling value for key %q: %w", SourcePrefix(src), k, err)
 			}
 
 			targetValue.SetMapIndex(reflect.ValueOf(k), nv.Elem())
@@ -410,11 +410,11 @@ func (m *Map[K, V]) decodeInto(target any) error {
 	// Copy all values that weren't non-inline fields into a temporary map.
 	// This is just to avoid mutating tm.
 	temp := NewMap[string, any](tm.Len())
-	tm.Range(func(k string, v any) error {
+	tm.Range(func(k string, v any, s *yaml.Node) error {
 		if _, outline := outlineKeys[k]; outline {
 			return nil
 		}
-		temp.Set(k, v)
+		temp.Set(k, v, s)
 		return nil
 	})
 
@@ -453,15 +453,15 @@ func (m *Map[K, V]) UnmarshalOrdered(src any) error {
 	}
 
 	var warns []error
-	if err := tsrc.Range(func(k string, v any) error {
+	if err := tsrc.Range(func(k string, v any, src *yaml.Node) error {
 		var dv V
 		err := Unmarshal(v, &dv)
 		if w := warning.As(err); w != nil {
-			warns = append(warns, w.Wrapf("while unmarshaling the value for key %q", k))
+			warns = append(warns, w.Wrapf("%swhile unmarshaling the value for key %q", SourcePrefix(src), k))
 		} else if err != nil {
-			return fmt.Errorf("unmarshaling value for key %q: %w", k, err)
+			return fmt.Errorf("%sunmarshaling value for key %q: %w", SourcePrefix(src), k, err)
 		}
-		tm.Set(k, dv)
+		tm.Set(k, dv, src)
 		return nil
 	}); err != nil {
 		return err

ordered/yaml.go

@@ -29,7 +29,7 @@ func decodeYAML(seen map[*yaml.Node]bool, n *yaml.Node) (any, error) {
 	// a: &a  // seen is empty on encoding a
 	//   b: *a   // seen contains a while encoding b
 	if seen[n] {
-		return nil, fmt.Errorf("line %d, col %d: infinite recursion", n.Line, n.Column)
+		return nil, fmt.Errorf("%sinfinite recursion", SourcePrefix(n))
 	}
 	seen[n] = true
 
@@ -78,7 +78,7 @@ func decodeYAML(seen map[*yaml.Node]bool, n *yaml.Node) (any, error) {
 			if err != nil {
 				return err
 			}
-			m.Set(key, v)
+			m.Set(key, v, val)
 			return nil
 		})
 		if err != nil {
@@ -98,11 +98,11 @@ func decodeYAML(seen map[*yaml.Node]bool, n *yaml.Node) (any, error) {
 		case 1:
 			return decodeYAML(seen, n.Content[0])
 		default:
-			return nil, fmt.Errorf("line %d, col %d: document contains more than 1 content item (%d)", n.Line, n.Column, len(n.Content))
+			return nil, fmt.Errorf("%sdocument contains more than 1 content item (%d)", SourcePrefix(n), len(n.Content))
 		}
 
 	default:
-		return nil, fmt.Errorf("line %d, col %d: unsupported kind %x", n.Line, n.Column, n.Kind)
+		return nil, fmt.Errorf("%sunsupported kind %x", SourcePrefix(n), n.Kind)
 	}
 }
 
@@ -138,7 +138,7 @@ func rangeYAMLMapImpl(merged map[*yaml.Node]bool, n *yaml.Node, f func(key strin
 		// gopkg.in/yaml.v3 parses mapping node contents as a flat list:
 		// key, value, key, value...
 		if len(n.Content)%2 != 0 {
-			return fmt.Errorf("line %d, col %d: mapping node has odd content length %d", n.Line, n.Column, len(n.Content))
+			return fmt.Errorf("%smapping node has odd content length %d", SourcePrefix(n), len(n.Content))
 		}
 
 		// Keys at an outer level take precedence over keys being merged:
@@ -224,7 +224,7 @@ func rangeYAMLMapImpl(merged map[*yaml.Node]bool, n *yaml.Node, f func(key strin
 
 	default:
 		// TODO: Use %v once yaml.Kind has a String method
-		return fmt.Errorf("line %d, col %d: cannot range over node kind %x", n.Line, n.Column, n.Kind)
+		return fmt.Errorf("%scannot range over node kind %x", SourcePrefix(n), n.Kind)
 	}
 	return nil
 }
@@ -245,7 +245,7 @@ func canonicalMapKey(n *yaml.Node) (string, error) {
 		}
 		if x == nil || n.Tag == "!!null" {
 			// Nulls are not valid JSON keys.
-			return "", fmt.Errorf("line %d, col %d: null not supported as a map key", n.Line, n.Column)
+			return "", fmt.Errorf("%snull is not supported as a map key", SourcePrefix(n))
 		}
 		switch n.Tag {
 		case "!!bool":
@@ -265,6 +265,15 @@ func canonicalMapKey(n *yaml.Node) (string, error) {
 
 	default:
 		// TODO: Use %v once yaml.Kind has a String method
-		return "", fmt.Errorf("line %d, col %d: cannot use node kind %x as a map key", n.Line, n.Column, n.Kind)
+		return "", fmt.Errorf("%scannot use node kind %x as a map key", SourcePrefix(n), n.Kind)
 	}
 }
+
+// SourcePrefix returns some info from a YAML node. If n is nil, it returns the
+// empty string. It is intended to prefix e.g. an error string.
+func SourcePrefix(n *yaml.Node) string {
+	if n == nil {
+		return ""
+	}
+	return fmt.Sprintf("at line %d, column %d: ", n.Line, n.Column)
+}

pipeline.go

@@ -7,6 +7,7 @@ import (
 	"github.com/buildkite/go-pipeline/ordered"
 	"github.com/buildkite/go-pipeline/warning"
 	"github.com/buildkite/interpolate"
+	"gopkg.in/yaml.v3"
 )
 
 // Pipeline models a pipeline.
@@ -112,7 +113,7 @@ func (p *Pipeline) Interpolate(interpolationEnv InterpolationEnv, preferRuntimeE
 // be interpolated into later environment variables, we also add the results
 // to interpolationEnv, making the input ordering of p.Env potentially important.
 func (p *Pipeline) interpolateEnvBlock(interpolationEnv InterpolationEnv, preferRuntimeEnv bool) error {
-	return p.Env.Range(func(k, v string) error {
+	return p.Env.Range(func(k, v string, src *yaml.Node) error {
 		// We interpolate both keys and values.
 		intk, err := interpolate.Interpolate(interpolationEnv, k)
 		if err != nil {
@@ -125,7 +126,7 @@ func (p *Pipeline) interpolateEnvBlock(interpolationEnv InterpolationEnv, prefer
 			return err
 		}
 
-		p.Env.Replace(k, intk, intv)
+		p.Env.Replace(k, intk, intv, src)
 
 		// If the variable already existed and we prefer the runtime environment then don't overwrite it
 		if _, exists := interpolationEnv.Get(intk); !(preferRuntimeEnv && exists) {

plugins.go

@@ -28,7 +28,7 @@ func (p *Plugins) UnmarshalOrdered(o any) error {
 	// part remains the same.
 	// Parse each "key: value" as "name: config", then append in order.
 	unmarshalMap := func(m *ordered.MapSA) error {
-		return m.Range(func(k string, v any) error {
+		return m.Range(func(k string, v any, _ *yaml.Node) error {
 			// ToMapRecursive demolishes any ordering within the plugin config.
 			// This is needed because the backend likes to reorder the keys,
 			// and for signing we need the JSON form to be stable.