Skip to content

Commit

Permalink
Merge pull request #11 from buildkite/unmarshal-aliases
Browse files Browse the repository at this point in the history 
Add aliases tag and use it
  • Loading branch information
@DrJosh9000
DrJosh9000 authored Nov 29, 2023
2 parents f5964d7 + e7ed78f commit d06cd5e
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 51 deletions.
26 changes: 18 additions & 8 deletions interpolate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ type selfInterpolater interface {
interpolate(stringTransformer) error
}

// interpolateAny interpolates (almost) anything in-place. When passed a string,
// it returns a new string. Anything it doesn't know how to interpolate is
// returned unaltered.
// interpolateAny interpolates most things, mostly in-place. When passed a
// string, it returns a new string. Anything it doesn't know how to interpolate
// is returned unaltered.
func interpolateAny[T any](tf stringTransformer, o T) (T, error) {
// The box-typeswitch-unbox dance is required because the Go compiler
// has no type switch for type parameters.
Expand All @@ -45,11 +45,7 @@ func interpolateAny[T any](tf stringTransformer, o T) (T, error) {
err = t.interpolate(tf)

case *string:
if t == nil {
return o, nil
}
*t, err = tf.Transform(*t)
a = t
err = interpolateString(tf, t)

case string:
a, err = tf.Transform(t)
Expand Down Expand Up @@ -85,6 +81,20 @@ func interpolateAny[T any](tf stringTransformer, o T) (T, error) {
return a.(T), err
}

// interpolateString is a helper to interpolate a string field in-place
// (requiring a pointer to the field).
func interpolateString(tf stringTransformer, p *string) error {
if p == nil {
return nil
}
s, err := tf.Transform(*p)
if err != nil {
return err
}
*p = s
return nil
}

// interpolateSlice applies interpolateAny over any type of slice. Values in the
// slice are updated in-place.
func interpolateSlice[E any, S ~[]E](tf stringTransformer, s S) error {
Expand Down
24 changes: 21 additions & 3 deletions ordered/unmarshal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ type Unmarshaler interface {
// yaml tags includes ",inline". Inline fields must themselves be a type
// that Unmarshal can unmarshal *Map[string, any] into - another struct or
// Map or map with string keys.
// Struct targets can also have `aliases` tags of the form
// `aliases:"apple,banana,citron"`
// If the field name or yaml tag key doesn't match, Unmarshal looks through
// the aliases list to see if any are present, and uses the value for the
// first.
// - S = []any (also recursively containing values with types from this list),
// which is recursively unmarshaled elementwise; D is *[]any or
// *[]somethingElse.
Expand Down Expand Up @@ -313,17 +318,30 @@ func (m *Map[K, V]) decodeInto(target any) error {
}

// Is there a value for this key?
v, has := tm.Get(key)
value, has := tm.Get(key)
if !has {
// Look for aliases, and choose the first with a value.
atag, _ := field.Tag.Lookup("aliases")
for _, alias := range strings.Split(atag, ",") {
value, has = tm.Get(alias)
if has {
key = alias
break
}
}
}
if !has {
// Couldn't find a value for the key or any aliases, so skip.
continue
}

// Now load v into this field.
// key matched a field, so it isn't inline.
outlineKeys[key] = struct{}{}

// Now load value into the field recursively.
// Get a pointer to the field. This works because target is a pointer.
ptrToField := innerValue.FieldByIndex(field.Index).Addr()
if err := Unmarshal(v, ptrToField.Interface()); err != nil {
if err := Unmarshal(value, ptrToField.Interface()); err != nil {
return err
}
}
Expand Down
10 changes: 10 additions & 0 deletions ordered/unmarshal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ type ordinaryStruct struct {
Count int
Fader float64
Slicey []int
Alias string `aliases:"synonym,alternate"`
hidden string

Next *ordinaryStruct
Expand Down Expand Up @@ -426,6 +427,7 @@ func TestUnmarshal(t *testing.T) {
TupleSA{Key: "count", Value: 42},
TupleSA{Key: "fader", Value: 2.71828},
TupleSA{Key: "slicey", Value: []any{5, 6, 7, 8}},
TupleSA{Key: "synonym", Value: "alias"},
),
dst: &ordinaryStruct{},
want: &ordinaryStruct{
Expand All @@ -435,6 +437,7 @@ func TestUnmarshal(t *testing.T) {
Count: 42,
Fader: 2.71828,
Slicey: []int{5, 6, 7, 8},
Alias: "alias",
},
},
{
Expand All @@ -448,6 +451,7 @@ func TestUnmarshal(t *testing.T) {
TupleSA{Key: "count", Value: 42},
TupleSA{Key: "fader", Value: 2.71828},
TupleSA{Key: "slicey", Value: []any{5, 6, 7, 8}},
TupleSA{Key: "alias", Value: "alias"},
TupleSA{Key: "notAField", Value: "super important"},
),
dst: &ordinaryStruct{},
Expand All @@ -458,6 +462,7 @@ func TestUnmarshal(t *testing.T) {
Count: 42,
Fader: 2.71828,
Slicey: []int{5, 6, 7, 8},
Alias: "alias",
Remaining: map[string]any{
"ignore": "YOU CANNOT DO THIS!!!",
"mountain": "actually we call them molehills here",
Expand All @@ -475,6 +480,7 @@ func TestUnmarshal(t *testing.T) {
TupleSA{Key: "count", Value: nil},
TupleSA{Key: "fader", Value: 2.71828},
TupleSA{Key: "slicey", Value: []any{5, 6, 7, 8}},
TupleSA{Key: "alternate", Value: "Agent Vaughn"},
TupleSA{Key: "hidden", Value: "no"},
TupleSA{Key: "notAField", Value: "super important"},
),
Expand All @@ -486,6 +492,7 @@ func TestUnmarshal(t *testing.T) {
Count: 69,
Fader: 3.14159,
Slicey: []int{1, 2, 3, 4},
Alias: "Sydney Bristow",
hidden: "yes",
Remaining: map[string]any{
"existing": "wombat",
Expand All @@ -499,6 +506,7 @@ func TestUnmarshal(t *testing.T) {
Count: 0, // nil becomes a SetZero call
Fader: 2.71828,
Slicey: []int{1, 2, 3, 4, 5, 6, 7, 8},
Alias: "Agent Vaughn",
hidden: "yes",
Remaining: map[string]any{
"existing": "wombat",
Expand All @@ -518,6 +526,7 @@ func TestUnmarshal(t *testing.T) {
TupleSA{Key: "fader", Value: 2.71828},
TupleSA{Key: "notAField", Value: "super important"},
TupleSA{Key: "slicey", Value: []any{5, 6, 7, 8}},
TupleSA{Key: "alias", Value: "JJ Abrams"},
TupleSA{Key: "inner", Value: MapFromItems(
TupleSA{Key: "llama", Value: "Kuzco"},
)},
Expand All @@ -537,6 +546,7 @@ func TestUnmarshal(t *testing.T) {
Count: 42,
Fader: 2.71828,
Slicey: []int{5, 6, 7, 8},
Alias: "JJ Abrams",
Inner: struct{ Llama string }{"Kuzco"},
Next: &ordinaryStruct{
Key: "another value",
Expand Down
28 changes: 11 additions & 17 deletions parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,12 @@ steps:
want := &Pipeline{
Steps: Steps{
&CommandStep{
Label: ":docker: building image",
Command: "docker build .",
RemainingFields: map[string]any{
"agents": ordered.MapFromItems(
ordered.TupleSA{Key: "queue", Value: "default"},
),
"name": ":docker: building image",
"type": "script",
"agent_query_rules": []any{"queue=default"},
},
Expand Down Expand Up @@ -148,7 +148,7 @@ steps:
"queue": "default"
},
"command": "docker build .",
"name": ":docker: building image",
"label": ":docker: building image",
"type": "script"
}
]
Expand Down Expand Up @@ -207,12 +207,12 @@ steps:
want := &Pipeline{
Steps: Steps{
&CommandStep{
Label: ":docker: building image",
Command: "docker build .",
RemainingFields: map[string]any{
"agents": ordered.MapFromItems(
ordered.TupleSA{Key: "queue", Value: "default"},
),
"name": ":docker: building image",
"type": "script",
"agent_query_rules": []any{"queue=default"},
},
Expand Down Expand Up @@ -256,7 +256,7 @@ steps:
"queue": "default"
},
"command": "docker build .",
"name": ":docker: building image",
"label": ":docker: building image",
"type": "script"
}
]
Expand Down Expand Up @@ -588,9 +588,7 @@ func TestParserParsesTopLevelSteps(t *testing.T) {
Steps: Steps{
&CommandStep{
Command: "echo hello world",
RemainingFields: map[string]any{
"name": "Build",
},
Label: "Build",
},
&WaitStep{Scalar: "wait"},
},
Expand All @@ -607,7 +605,7 @@ func TestParserParsesTopLevelSteps(t *testing.T) {
"steps": [
{
"command": "echo hello world",
"name": "Build"
"label": "Build"
},
"wait"
]
Expand Down Expand Up @@ -925,6 +923,7 @@ steps:
),
Steps: Steps{
&CommandStep{
Label: ":docker: Docker Build",
Command: "echo foo",
Plugins: Plugins{
{
Expand All @@ -935,9 +934,6 @@ steps:
},
},
},
RemainingFields: map[string]any{
"label": string(":docker: Docker Build"),
},
},
},
}
Expand Down Expand Up @@ -1013,6 +1009,7 @@ steps:
want := &Pipeline{
Steps: Steps{
&CommandStep{
Label: ":s3: xxx",
Command: "script/buildkite/xxx.sh",
Plugins: Plugins{
{
Expand Down Expand Up @@ -1041,7 +1038,6 @@ steps:
},
},
RemainingFields: map[string]any{
"name": ":s3: xxx",
"agents": ordered.MapFromItems(
ordered.TupleSA{Key: "queue", Value: "xxx"},
),
Expand All @@ -1064,7 +1060,7 @@ steps:
"queue": "xxx"
},
"command": "script/buildkite/xxx.sh",
"name": ":s3: xxx",
"label": ":s3: xxx",
"plugins": [
{
"github.com/xxx/aws-assume-role-buildkite-plugin#v0.1.0": {
Expand Down Expand Up @@ -1118,6 +1114,7 @@ func TestParserParsesScalarPlugins(t *testing.T) {
want := &Pipeline{
Steps: Steps{
&CommandStep{
Label: ":s3: xxx",
Command: "script/buildkite/xxx.sh",
Plugins: Plugins{
{
Expand All @@ -1133,9 +1130,6 @@ func TestParserParsesScalarPlugins(t *testing.T) {
},
},
},
RemainingFields: map[string]any{
"name": ":s3: xxx",
},
},
},
}
Expand All @@ -1151,7 +1145,7 @@ func TestParserParsesScalarPlugins(t *testing.T) {
"steps": [
{
"command": "script/buildkite/xxx.sh",
"name": ":s3: xxx",
"label": ":s3: xxx",
"plugins": [
{
"github.com/buildkite-plugins/example-plugin-buildkite-plugin#v1.0.0": null
Expand Down
40 changes: 24 additions & 16 deletions step_command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ type Signature struct {
//
// Standard caveats apply - see the package comment.
type CommandStep struct {
// Fields common to various step types
Key string `yaml:"key,omitempty" aliases:"id,identifier"`
Label string `yaml:"label,omitempty" aliases:"name"`

// Fields that are meaningful specifically for command steps
Command string `yaml:"command"`
Plugins Plugins `yaml:"plugins,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
Expand Down Expand Up @@ -59,8 +64,7 @@ func (c *CommandStep) UnmarshalOrdered(src any) error {
type wrappedCommand CommandStep
// Unmarshal into this secret type, then process special fields specially.
fullCommand := new(struct {
Command []string `yaml:"command"`
Commands []string `yaml:"commands"`
Commands []string `yaml:"commands" aliases:"command"`

// Use inline trickery to capture the rest of the struct.
Rem *wrappedCommand `yaml:",inline"`
Expand All @@ -75,8 +79,7 @@ func (c *CommandStep) UnmarshalOrdered(src any) error {
// string consistently than it is to pick apart multiple strings
// in a consistent way in order to hash all of them
// consistently.
cmds := append(fullCommand.Command, fullCommand.Commands...)
c.Command = strings.Join(cmds, "\n")
c.Command = strings.Join(fullCommand.Commands, "\n")
return nil
}

Expand All @@ -94,36 +97,41 @@ func (c *CommandStep) InterpolateMatrixPermutation(mp MatrixPermutation) error {
}

func (c *CommandStep) interpolate(tf stringTransformer) error {
cmd, err := tf.Transform(c.Command)
if err != nil {
return err
// Fields that are interpolated with env vars and matrix tokens:
// command, plugins
if err := interpolateString(tf, &c.Command); err != nil {
return fmt.Errorf("interpolating command: %w", err)
}
c.Command = cmd

if err := interpolateSlice(tf, c.Plugins); err != nil {
return err
return fmt.Errorf("interpolating plugins: %w", err)
}

switch tf.(type) {
case envInterpolator:
// Env interpolation applies to nearly everything:
// key, depends_on, env (keys and values), matrix
if err := interpolateString(tf, &c.Key); err != nil {
return fmt.Errorf("interpolating key: %w", err)
}
if err := interpolateMap(tf, c.Env); err != nil {
return err
return fmt.Errorf("interpolating env: %w", err)
}
if c.Matrix, err = interpolateAny(tf, c.Matrix); err != nil {
return err
if err := c.Matrix.interpolate(tf); err != nil {
return fmt.Errorf("interpolating matrix: %w", err)
}

case matrixInterpolator:
// Matrix interpolation doesn't apply to env keys.
// Matrix interpolation applies only to some things, but particularly
// only affects env values (not env keys).
if err := interpolateMapValues(tf, c.Env); err != nil {
return err
return fmt.Errorf("interpolating env values: %w", err)
}
}

// NB: Do not interpolate Signature.

if err := interpolateMap(tf, c.RemainingFields); err != nil {
return err
return fmt.Errorf("interpolating remaining fields: %w", err)
}

return nil
Expand Down
Loading

0 comments on commit d06cd5e

Please sign in to comment.