Introduce TestViaRPC to converge parser and rpcparser's implementatio…

…ns (#108)

* Switch parser from gopkg.in/yaml.v2 to github.com/goccy/go-yaml

* Regex nameables should ensure that the whole string is matched

* Insert clone command even if no commands were generated for a task

* Handle timeout_in field

* Introduce TestViaRPC to converge parser and rpcparser's implementations

* TestViaRPC: load expected tasks from JSON fixtures

Also automatically create new JSON fixtures if missing
to aid in migration.

* Move some TestViaRPC()'s functionality into separate functions

* TasksToJSON: include newline at the end of file

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/go-test/deep v1.0.7
+	github.com/goccy/go-yaml v1.8.2
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-github/v32 v32.1.0

go.sum

@@ -76,6 +76,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -96,11 +98,17 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
 github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
 github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
+github.com/goccy/go-yaml v1.8.2 h1:gDYrSN12XK/wQTFjxWIgcIqjNCV/Zb5V09M7cq+dbCs=
+github.com/goccy/go-yaml v1.8.2/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -183,6 +191,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c h1:pGh5EFIfczeDHwgMHgfwjhZzL+8/E3uZF6T7vER/W8c=
 github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c/go.mod h1:xw2Gm4Mg+ST9s8fHR1VkUIyOJMJnSloRZlPQB+wyVpY=
 github.com/lestrrat-go/jsref v0.0.0-20181205001954-1b590508f37d h1:1eeFdKL5ySmmYevvKv7iECIc4dTATeKTtBqP4/nXxDk=
@@ -197,6 +207,11 @@ github.com/lestrrat-go/structinfo v0.0.0-20190212233437-acd51874663b h1:YUFRoeHK
 github.com/lestrrat-go/structinfo v0.0.0-20190212233437-acd51874663b/go.mod h1:s2U6PowV3/Jobkx/S9d0XiPwOzs6niW3DIouw+7nZC8=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -357,11 +372,14 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -434,6 +452,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v9 v9.30.0 h1:Wk0Z37oBmKj9/n+tPyBHZmeL19LaCoK3Qq48VwYENss=
+gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

internal/testutil/api.go

@@ -30,5 +30,32 @@ func TasksToJSON(t *testing.T, tasks []*api.Task) []byte {
 		t.Fatal(err)
 	}
 
+	res = append(res, '\n')
+
 	return res
 }
+
+func TasksFromJSON(t *testing.T, jsonBytes []byte) (result []*api.Task) {
+	var jsonTasks []interface{}
+
+	if err := json.Unmarshal(jsonBytes, &jsonTasks); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, jsonTask := range jsonTasks {
+		jsonTaskBytes, err := json.Marshal(jsonTask)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var task api.Task
+
+		if err := protojson.Unmarshal(jsonTaskBytes, &task); err != nil {
+			t.Fatal(err)
+		}
+
+		result = append(result, &task)
+	}
+
+	return
+}

pkg/parser/instance/resources.go

@@ -47,7 +47,7 @@ func ParseMegaBytes(s string) (uint32, error) {
 	case "g":
 		memoryResult *= kibi
 	default:
-		return 0, fmt.Errorf("%w: unsupported suffix: '%s'", parsererror.ErrParsing, suffixPart)
+		return 0, fmt.Errorf("%w: unsupported digital information unit suffix: '%s'", parsererror.ErrParsing, suffixPart)
 	}
 
 	return memoryResult, nil

pkg/parser/modifier/matrix/deepcopier.go

@@ -3,11 +3,11 @@ package matrix
 import (
 	"bytes"
 	"encoding/gob"
-	"gopkg.in/yaml.v2"
+	"github.com/goccy/go-yaml"
 )
 
 // This is rather inefficient and error-prone (due to the need to manually register unknown types),
-// but nevertheless works flawlessly for yaml.v2 structures, compared to other alternatives.
+// but nevertheless works flawlessly for YAML structures, compared to other alternatives.
 func deepcopy(dst, src interface{}) error {
 	// Register unknown types
 	// https://golang.org/pkg/encoding/gob/#Register

pkg/parser/modifier/matrix/expander.go

@@ -1,7 +1,7 @@
 package matrix
 
 import (
-	"gopkg.in/yaml.v2"
+	"github.com/goccy/go-yaml"
 )
 
 // Callback function to be called by traverse().

pkg/parser/modifier/matrix/matrix.go

@@ -2,7 +2,7 @@ package matrix
 
 import (
 	"errors"
-	"gopkg.in/yaml.v2"
+	"github.com/goccy/go-yaml"
 	"strings"
 )
 

pkg/parser/modifier/matrix/matrix_test.go

@@ -3,8 +3,8 @@ package matrix_test
 import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/modifier/matrix"
 	"github.com/go-test/deep"
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -21,7 +21,7 @@ func getDocument(t *testing.T, path string, index int) string {
 	}
 	defer file.Close()
 
-	decoder := yaml.NewDecoder(file)
+	decoder := yaml.NewDecoder(file, yaml.UseOrderedMap())
 	var document yaml.MapSlice
 
 	for i := 0; i < index; i++ {
@@ -40,7 +40,7 @@ func getDocument(t *testing.T, path string, index int) string {
 
 // Unmarshals YAML specified by yamlText to a yaml.MapSlice to simplify comparison.
 func yamlAsStruct(t *testing.T, yamlText string) (result yaml.MapSlice) {
-	if err := yaml.Unmarshal([]byte(yamlText), &result); err != nil {
+	if err := yaml.UnmarshalWithOptions([]byte(yamlText), &result, yaml.UseOrderedMap()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -82,7 +82,7 @@ var badCases = []struct {
 
 func runPreprocessor(input string) (string, error) {
 	var tree yaml.MapSlice
-	err := yaml.Unmarshal([]byte(input), &tree)
+	err := yaml.UnmarshalWithOptions([]byte(input), &tree, yaml.UseOrderedMap())
 	if err != nil {
 		return "", err
 	}

pkg/parser/modifier/matrix/testdata/empty.yaml

@@ -1,2 +1,3 @@
 {}
 ---
+{}

pkg/parser/nameable/regex_test.go

@@ -18,7 +18,7 @@ func TestRegexNameable(t *testing.T) {
 func TestFirstGroupOrDefault(t *testing.T) {
 	const defaultValue = "main"
 
-	name := nameable.NewRegexNameable("(.*)task")
+	name := nameable.NewRegexNameable("^(.*)task$")
 	assert.Equal(t, defaultValue, name.FirstGroupOrDefault("123", defaultValue))
 	assert.Equal(t, defaultValue, name.FirstGroupOrDefault("task", defaultValue))
 	assert.Equal(t, "a", name.FirstGroupOrDefault("a_task", defaultValue))

pkg/parser/node/accessor_test.go

@@ -2,9 +2,9 @@ package node_test
 
 import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/node"
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
 	"testing"
 )
 

pkg/parser/node/finder_test.go

@@ -2,8 +2,8 @@ package node_test
 
 import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/node"
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
 	"testing"
 )
 

pkg/parser/node/node.go

@@ -3,7 +3,7 @@ package node
 import (
 	"errors"
 	"fmt"
-	"gopkg.in/yaml.v2"
+	"github.com/goccy/go-yaml"
 )
 
 type Node struct {

pkg/parser/node/node_test.go

@@ -2,8 +2,8 @@ package node_test
 
 import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/node"
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
 	"testing"
 )
 

pkg/parser/parser.go

@@ -13,9 +13,9 @@ import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/parseable"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/parsererror"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/task"
+	"github.com/goccy/go-yaml"
 	"github.com/golang/protobuf/ptypes"
 	"github.com/lestrrat-go/jsschema"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"path/filepath"
 	"regexp"
@@ -55,8 +55,8 @@ func New(opts ...Option) *Parser {
 
 	// Register parsers
 	parser.parsers = map[nameable.Nameable]parseable.Parseable{
-		nameable.NewRegexNameable("(.*)task"): &task.Task{},
-		nameable.NewRegexNameable("(.*)pipe"): &task.DockerPipe{},
+		nameable.NewRegexNameable("^(.*)task$"): &task.Task{},
+		nameable.NewRegexNameable("^(.*)pipe$"): &task.DockerPipe{},
 	}
 
 	return parser
@@ -109,7 +109,7 @@ func (p *Parser) Parse(config string) (*Result, error) {
 	var parsed yaml.MapSlice
 
 	// Unmarshal YAML
-	if err := yaml.Unmarshal([]byte(config), &parsed); err != nil {
+	if err := yaml.UnmarshalWithOptions([]byte(config), &parsed, yaml.UseOrderedMap()); err != nil {
 		return nil, err
 	}
 
@@ -355,10 +355,6 @@ func (p *Parser) createServiceTasks(protoTasks []*api.Task) ([]*api.Task, error)
 }
 
 func ensureCloneInstruction(task *api.Task) {
-	if len(task.Commands) == 0 {
-		return
-	}
-
 	for _, command := range task.Commands {
 		if command.Name == "clone" {
 			return
@@ -368,12 +364,15 @@ func ensureCloneInstruction(task *api.Task) {
 	// Inherit "image" property from the first task (if any),
 	// or otherwise we might break Docker Pipe
 	var properties map[string]string
-	image, ok := task.Commands[0].Properties["image"]
-	if ok {
-		properties = map[string]string{
-			"image": image,
+
+	if len(task.Commands) != 0 {
+		image, ok := task.Commands[0].Properties["image"]
+		if ok {
+			properties = map[string]string{
+				"image": image,
+			}
+			delete(task.Commands[0].Properties, "image")
 		}
-		delete(task.Commands[0].Properties, "image")
 	}
 
 	cloneCommand := &api.Command{