diff --git a/go.mod b/go.mod index a97d61c3..933ffb9e 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-test/deep v1.0.7 + github.com/goccy/go-yaml v1.8.2 github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/protobuf v1.4.2 github.com/google/go-github/v32 v32.1.0 diff --git a/go.sum b/go.sum index 77b0cb67..bb490558 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -96,11 +98,17 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/goccy/go-yaml v1.8.2 h1:gDYrSN12XK/wQTFjxWIgcIqjNCV/Zb5V09M7cq+dbCs= +github.com/goccy/go-yaml v1.8.2/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -183,6 +191,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c h1:pGh5EFIfczeDHwgMHgfwjhZzL+8/E3uZF6T7vER/W8c= github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c/go.mod h1:xw2Gm4Mg+ST9s8fHR1VkUIyOJMJnSloRZlPQB+wyVpY= github.com/lestrrat-go/jsref v0.0.0-20181205001954-1b590508f37d h1:1eeFdKL5ySmmYevvKv7iECIc4dTATeKTtBqP4/nXxDk= @@ -197,6 +207,11 @@ github.com/lestrrat-go/structinfo v0.0.0-20190212233437-acd51874663b h1:YUFRoeHK github.com/lestrrat-go/structinfo v0.0.0-20190212233437-acd51874663b/go.mod h1:s2U6PowV3/Jobkx/S9d0XiPwOzs6niW3DIouw+7nZC8= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -357,11 +372,14 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -434,6 +452,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.30.0 h1:Wk0Z37oBmKj9/n+tPyBHZmeL19LaCoK3Qq48VwYENss= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/internal/testutil/api.go b/internal/testutil/api.go index 1c277acc..fd32d8de 100644 --- a/internal/testutil/api.go +++ b/internal/testutil/api.go @@ -30,5 +30,32 @@ func TasksToJSON(t *testing.T, tasks []*api.Task) []byte { t.Fatal(err) } + res = append(res, '\n') + return res } + +func TasksFromJSON(t *testing.T, jsonBytes []byte) (result []*api.Task) { + var jsonTasks []interface{} + + if err := json.Unmarshal(jsonBytes, &jsonTasks); err != nil { + t.Fatal(err) + } + + for _, jsonTask := range jsonTasks { + jsonTaskBytes, err := json.Marshal(jsonTask) + if err != nil { + t.Fatal(err) + } + + var task api.Task + + if err := protojson.Unmarshal(jsonTaskBytes, &task); err != nil { + t.Fatal(err) + } + + result = append(result, &task) + } + + return +} diff --git a/pkg/parser/instance/resources.go b/pkg/parser/instance/resources.go index fda344f1..862bf804 100644 --- a/pkg/parser/instance/resources.go +++ b/pkg/parser/instance/resources.go @@ -47,7 +47,7 @@ func ParseMegaBytes(s string) (uint32, error) { case "g": memoryResult *= kibi default: - return 0, fmt.Errorf("%w: unsupported suffix: '%s'", parsererror.ErrParsing, suffixPart) + return 0, fmt.Errorf("%w: unsupported digital information unit suffix: '%s'", parsererror.ErrParsing, suffixPart) } return memoryResult, nil diff --git a/pkg/parser/modifier/matrix/deepcopier.go b/pkg/parser/modifier/matrix/deepcopier.go index dfcb576f..6c052f38 100644 --- a/pkg/parser/modifier/matrix/deepcopier.go +++ b/pkg/parser/modifier/matrix/deepcopier.go @@ -3,11 +3,11 @@ package matrix import ( "bytes" "encoding/gob" - "gopkg.in/yaml.v2" + "github.com/goccy/go-yaml" ) // This is rather inefficient and error-prone (due to the need to manually register unknown types), -// but nevertheless works flawlessly for yaml.v2 structures, compared to other alternatives. +// but nevertheless works flawlessly for YAML structures, compared to other alternatives. func deepcopy(dst, src interface{}) error { // Register unknown types // https://golang.org/pkg/encoding/gob/#Register diff --git a/pkg/parser/modifier/matrix/expander.go b/pkg/parser/modifier/matrix/expander.go index 74e6c277..006f37ce 100644 --- a/pkg/parser/modifier/matrix/expander.go +++ b/pkg/parser/modifier/matrix/expander.go @@ -1,7 +1,7 @@ package matrix import ( - "gopkg.in/yaml.v2" + "github.com/goccy/go-yaml" ) // Callback function to be called by traverse(). diff --git a/pkg/parser/modifier/matrix/matrix.go b/pkg/parser/modifier/matrix/matrix.go index 6bd34f18..5105b99c 100644 --- a/pkg/parser/modifier/matrix/matrix.go +++ b/pkg/parser/modifier/matrix/matrix.go @@ -2,7 +2,7 @@ package matrix import ( "errors" - "gopkg.in/yaml.v2" + "github.com/goccy/go-yaml" "strings" ) diff --git a/pkg/parser/modifier/matrix/matrix_test.go b/pkg/parser/modifier/matrix/matrix_test.go index 125ec65b..6b18c1e7 100644 --- a/pkg/parser/modifier/matrix/matrix_test.go +++ b/pkg/parser/modifier/matrix/matrix_test.go @@ -3,8 +3,8 @@ package matrix_test import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/modifier/matrix" "github.com/go-test/deep" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" "io/ioutil" "os" "path/filepath" @@ -21,7 +21,7 @@ func getDocument(t *testing.T, path string, index int) string { } defer file.Close() - decoder := yaml.NewDecoder(file) + decoder := yaml.NewDecoder(file, yaml.UseOrderedMap()) var document yaml.MapSlice for i := 0; i < index; i++ { @@ -40,7 +40,7 @@ func getDocument(t *testing.T, path string, index int) string { // Unmarshals YAML specified by yamlText to a yaml.MapSlice to simplify comparison. func yamlAsStruct(t *testing.T, yamlText string) (result yaml.MapSlice) { - if err := yaml.Unmarshal([]byte(yamlText), &result); err != nil { + if err := yaml.UnmarshalWithOptions([]byte(yamlText), &result, yaml.UseOrderedMap()); err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ var badCases = []struct { func runPreprocessor(input string) (string, error) { var tree yaml.MapSlice - err := yaml.Unmarshal([]byte(input), &tree) + err := yaml.UnmarshalWithOptions([]byte(input), &tree, yaml.UseOrderedMap()) if err != nil { return "", err } diff --git a/pkg/parser/modifier/matrix/testdata/empty.yaml b/pkg/parser/modifier/matrix/testdata/empty.yaml index 10637b5e..3064fe98 100644 --- a/pkg/parser/modifier/matrix/testdata/empty.yaml +++ b/pkg/parser/modifier/matrix/testdata/empty.yaml @@ -1,2 +1,3 @@ {} --- +{} diff --git a/pkg/parser/nameable/regex_test.go b/pkg/parser/nameable/regex_test.go index aac2770f..74498daf 100644 --- a/pkg/parser/nameable/regex_test.go +++ b/pkg/parser/nameable/regex_test.go @@ -18,7 +18,7 @@ func TestRegexNameable(t *testing.T) { func TestFirstGroupOrDefault(t *testing.T) { const defaultValue = "main" - name := nameable.NewRegexNameable("(.*)task") + name := nameable.NewRegexNameable("^(.*)task$") assert.Equal(t, defaultValue, name.FirstGroupOrDefault("123", defaultValue)) assert.Equal(t, defaultValue, name.FirstGroupOrDefault("task", defaultValue)) assert.Equal(t, "a", name.FirstGroupOrDefault("a_task", defaultValue)) diff --git a/pkg/parser/node/accessor_test.go b/pkg/parser/node/accessor_test.go index 17b39e0f..e8d06492 100644 --- a/pkg/parser/node/accessor_test.go +++ b/pkg/parser/node/accessor_test.go @@ -2,9 +2,9 @@ package node_test import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/node" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" "testing" ) diff --git a/pkg/parser/node/finder_test.go b/pkg/parser/node/finder_test.go index a425042c..49d62211 100644 --- a/pkg/parser/node/finder_test.go +++ b/pkg/parser/node/finder_test.go @@ -2,8 +2,8 @@ package node_test import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/node" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" "testing" ) diff --git a/pkg/parser/node/node.go b/pkg/parser/node/node.go index 6b30b7be..ea6b63e1 100644 --- a/pkg/parser/node/node.go +++ b/pkg/parser/node/node.go @@ -3,7 +3,7 @@ package node import ( "errors" "fmt" - "gopkg.in/yaml.v2" + "github.com/goccy/go-yaml" ) type Node struct { diff --git a/pkg/parser/node/node_test.go b/pkg/parser/node/node_test.go index ef32f3a3..f51efe03 100644 --- a/pkg/parser/node/node_test.go +++ b/pkg/parser/node/node_test.go @@ -2,8 +2,8 @@ package node_test import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/node" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" "testing" ) diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 7a7d0c6f..5e4cb1dc 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -13,9 +13,9 @@ import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/parseable" "github.com/cirruslabs/cirrus-cli/pkg/parser/parsererror" "github.com/cirruslabs/cirrus-cli/pkg/parser/task" + "github.com/goccy/go-yaml" "github.com/golang/protobuf/ptypes" "github.com/lestrrat-go/jsschema" - "gopkg.in/yaml.v2" "io/ioutil" "path/filepath" "regexp" @@ -55,8 +55,8 @@ func New(opts ...Option) *Parser { // Register parsers parser.parsers = map[nameable.Nameable]parseable.Parseable{ - nameable.NewRegexNameable("(.*)task"): &task.Task{}, - nameable.NewRegexNameable("(.*)pipe"): &task.DockerPipe{}, + nameable.NewRegexNameable("^(.*)task$"): &task.Task{}, + nameable.NewRegexNameable("^(.*)pipe$"): &task.DockerPipe{}, } return parser @@ -109,7 +109,7 @@ func (p *Parser) Parse(config string) (*Result, error) { var parsed yaml.MapSlice // Unmarshal YAML - if err := yaml.Unmarshal([]byte(config), &parsed); err != nil { + if err := yaml.UnmarshalWithOptions([]byte(config), &parsed, yaml.UseOrderedMap()); err != nil { return nil, err } @@ -355,10 +355,6 @@ func (p *Parser) createServiceTasks(protoTasks []*api.Task) ([]*api.Task, error) } func ensureCloneInstruction(task *api.Task) { - if len(task.Commands) == 0 { - return - } - for _, command := range task.Commands { if command.Name == "clone" { return @@ -368,12 +364,15 @@ func ensureCloneInstruction(task *api.Task) { // Inherit "image" property from the first task (if any), // or otherwise we might break Docker Pipe var properties map[string]string - image, ok := task.Commands[0].Properties["image"] - if ok { - properties = map[string]string{ - "image": image, + + if len(task.Commands) != 0 { + image, ok := task.Commands[0].Properties["image"] + if ok { + properties = map[string]string{ + "image": image, + } + delete(task.Commands[0].Properties, "image") } - delete(task.Commands[0].Properties, "image") } cloneCommand := &api.Command{ diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index fc9c237b..0e510a24 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -2,11 +2,16 @@ package parser_test import ( "encoding/json" + "errors" "fmt" "github.com/cirruslabs/cirrus-cli/internal/testutil" + "github.com/cirruslabs/cirrus-cli/pkg/rpcparser" + "github.com/go-test/deep" "github.com/stretchr/testify/require" "io/ioutil" + "os" "path/filepath" + "strings" "testing" "github.com/cirruslabs/cirrus-cli/pkg/parser" @@ -63,6 +68,82 @@ func TestInvalidConfigs(t *testing.T) { } } +// TestViaRPC ensures that the parser produces results identical to rpcparser. +func TestViaRPC(t *testing.T) { + cloudDir := absolutize("via-rpc") + + fileInfos, err := ioutil.ReadDir(cloudDir) + if err != nil { + t.Fatal(err) + } + + for _, fileInfo := range fileInfos { + fileInfo := fileInfo + + if !strings.HasSuffix(fileInfo.Name(), ".yml") { + continue + } + + t.Run(fileInfo.Name(), func(t *testing.T) { + viaRPCRunSingle(t, cloudDir, fileInfo.Name()) + }) + } +} + +func viaRPCRunSingle(t *testing.T, cloudDir string, yamlConfigName string) { + yamlConfigPath := filepath.Join(cloudDir, yamlConfigName) + fixtureName := strings.TrimSuffix(yamlConfigName, filepath.Ext(yamlConfigName)) + ".json" + fixturePath := filepath.Join(cloudDir, fixtureName) + + // Obtain expected result by loading JSON fixture + fixtureBytes, err := ioutil.ReadFile(fixturePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + viaRPCCreateJSONFixture(t, yamlConfigPath, fixturePath) + t.Fatalf("created new fixture: %s, don't forget to commit it", fixturePath) + } + + t.Fatal(err) + } + + fixtureTasks := testutil.TasksFromJSON(t, fixtureBytes) + + // Obtain the actual result by parsing YAML configuration using the local parser + localParser := parser.New() + localResult, err := localParser.ParseFromFile(yamlConfigPath) + if err != nil { + t.Fatal(err) + } + if len(localResult.Errors) != 0 { + t.Fatal(localResult.Errors) + } + + differences := deep.Equal(fixtureTasks, localResult.Tasks) + for _, difference := range differences { + fmt.Println(difference) + } + if len(differences) != 0 { + t.Fatal("found differences") + } +} + +func viaRPCCreateJSONFixture(t *testing.T, yamlConfigPath string, fixturePath string) { + // Aid in migration by automatically creating new JSON fixture using the RPC parser + rpcParser := rpcparser.Parser{} + rpcResult, err := rpcParser.ParseFromFile(yamlConfigPath) + if err != nil { + t.Fatal(err) + } + if len(rpcResult.Errors) != 0 { + t.Fatal(rpcResult.Errors) + } + + fixtureBytes := testutil.TasksToJSON(t, rpcResult.Tasks) + if err := ioutil.WriteFile(fixturePath, fixtureBytes, 0600); err != nil { + t.Fatal(err) + } +} + func TestSchema(t *testing.T) { p := parser.New() diff --git a/pkg/parser/task/behavior.go b/pkg/parser/task/behavior.go index 0da2513c..3049d99f 100644 --- a/pkg/parser/task/behavior.go +++ b/pkg/parser/task/behavior.go @@ -17,7 +17,7 @@ type Behavior struct { func NewBehavior() *Behavior { b := &Behavior{} - scriptNameable := nameable.NewRegexNameable("(.*)script") + scriptNameable := nameable.NewRegexNameable("^(.*)script$") b.OptionalField(scriptNameable, schema.TodoSchema, func(node *node.Node) error { command, err := handleScript(node, scriptNameable) if err != nil { diff --git a/pkg/parser/task/command.go b/pkg/parser/task/command.go index 3ca889ae..27ac5728 100644 --- a/pkg/parser/task/command.go +++ b/pkg/parser/task/command.go @@ -56,7 +56,7 @@ func (cache *CacheCommand) Parse(node *node.Node) error { return err } - cacheNameable := nameable.NewRegexNameable("(.*)cache") + cacheNameable := nameable.NewRegexNameable("^(.*)cache$") cache.proto.Name = cacheNameable.FirstGroupOrDefault(node.Name, "main") return nil diff --git a/pkg/parser/task/handler.go b/pkg/parser/task/handler.go index 1a3c9a04..1a867aa5 100644 --- a/pkg/parser/task/handler.go +++ b/pkg/parser/task/handler.go @@ -5,6 +5,7 @@ import ( "github.com/cirruslabs/cirrus-cli/pkg/parser/boolevator" "github.com/cirruslabs/cirrus-cli/pkg/parser/nameable" "github.com/cirruslabs/cirrus-cli/pkg/parser/node" + "strconv" ) func handleBoolevatorField(node *node.Node, mergedEnv map[string]string) (bool, error) { @@ -52,3 +53,18 @@ func handleScript(node *node.Node, nameable *nameable.RegexNameable) (*api.Comma }, }, nil } + +func handleTimeoutIn(node *node.Node, mergedEnv map[string]string) (string, error) { + timeoutHumanized, err := node.GetExpandedStringValue(mergedEnv) + if err != nil { + return "", err + } + + // Convert "humanized" value to seconds + timeoutSeconds, err := ParseSeconds(timeoutHumanized) + if err != nil { + return "", err + } + + return strconv.FormatUint(uint64(timeoutSeconds), 10), nil +} diff --git a/pkg/parser/task/pipe.go b/pkg/parser/task/pipe.go index 54de71d5..850e2879 100644 --- a/pkg/parser/task/pipe.go +++ b/pkg/parser/task/pipe.go @@ -85,6 +85,17 @@ func NewDockerPipe(env map[string]string) *DockerPipe { return nil }) + pipe.CollectibleField("timeout_in", schema.TodoSchema, func(node *node.Node) error { + timeoutInSeconds, err := handleTimeoutIn(node, environment.Merge(pipe.proto.Environment, env)) + if err != nil { + return err + } + + pipe.proto.Metadata.Properties["timeoutInSeconds"] = timeoutInSeconds + + return nil + }) + return pipe } diff --git a/pkg/parser/task/pipestep.go b/pkg/parser/task/pipestep.go index 4de990bb..c3f97680 100644 --- a/pkg/parser/task/pipestep.go +++ b/pkg/parser/task/pipestep.go @@ -31,7 +31,7 @@ func NewPipeStep(mergedEnv map[string]string) *PipeStep { return nil }) - scriptNameable := nameable.NewRegexNameable("(.*)script") + scriptNameable := nameable.NewRegexNameable("^(.*)script$") step.OptionalField(scriptNameable, schema.TodoSchema, func(node *node.Node) error { scripts, err := node.GetSliceOfNonEmptyStrings() if err != nil { diff --git a/pkg/parser/task/seconds.go b/pkg/parser/task/seconds.go new file mode 100644 index 00000000..44cde594 --- /dev/null +++ b/pkg/parser/task/seconds.go @@ -0,0 +1,40 @@ +package task + +import ( + "fmt" + "github.com/cirruslabs/cirrus-cli/pkg/parser/parsererror" + "strconv" + "strings" + "unicode" +) + +func ParseSeconds(s string) (uint32, error) { + // Split the string into two parts + sLower := strings.ToLower(s) + cutAfter := strings.LastIndexFunc(sLower, unicode.IsDigit) + digitsPart := sLower[:cutAfter+1] + suffixPart := sLower[cutAfter+1:] + + // Parse the digits part + parsedDigitsPartU64, err := strconv.ParseUint(digitsPart, 10, 32) + if err != nil { + return 0, err + } + parsedDigitsPartU32 := uint32(parsedDigitsPartU64) + + // Modify the digits part depending on the suffix part + switch suffixPart { + case "h": + parsedDigitsPartU32 *= 3600 + case "": + fallthrough + case "m": + parsedDigitsPartU32 *= 60 + case "s": + // nothing to do + default: + return 0, fmt.Errorf("%w: unsupported time unit suffix: '%s'", parsererror.ErrParsing, suffixPart) + } + + return parsedDigitsPartU32, nil +} diff --git a/pkg/parser/task/seconds_test.go b/pkg/parser/task/seconds_test.go new file mode 100644 index 00000000..64e95f30 --- /dev/null +++ b/pkg/parser/task/seconds_test.go @@ -0,0 +1,31 @@ +package task_test + +import ( + "github.com/cirruslabs/cirrus-cli/pkg/parser/task" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func parseSecondsHelper(t *testing.T, s string) uint32 { + result, err := task.ParseSeconds(s) + if err != nil { + t.Fatal(err) + } + return result +} + +func TestParseSeconds(t *testing.T) { + assert.EqualValues(t, (1 * time.Second).Seconds(), parseSecondsHelper(t, "1s")) + assert.EqualValues(t, (60 * time.Second).Seconds(), parseSecondsHelper(t, "60s")) + + assert.EqualValues(t, (0 * time.Minute).Seconds(), parseSecondsHelper(t, "0")) + assert.EqualValues(t, (1 * time.Minute).Seconds(), parseSecondsHelper(t, "1")) + assert.EqualValues(t, (60 * time.Minute).Seconds(), parseSecondsHelper(t, "60")) + + assert.EqualValues(t, (1 * time.Minute).Seconds(), parseSecondsHelper(t, "1m")) + assert.EqualValues(t, (5 * time.Minute).Seconds(), parseSecondsHelper(t, "5m")) + + assert.EqualValues(t, (1 * time.Hour).Seconds(), parseSecondsHelper(t, "1h")) + assert.EqualValues(t, (12 * time.Hour).Seconds(), parseSecondsHelper(t, "12h")) +} diff --git a/pkg/parser/task/task.go b/pkg/parser/task/task.go index ff3e9fc2..2268f791 100644 --- a/pkg/parser/task/task.go +++ b/pkg/parser/task/task.go @@ -87,7 +87,7 @@ func NewTask(env map[string]string) *Task { return nil }) - bgNameable := nameable.NewRegexNameable("(.*)background_script") + bgNameable := nameable.NewRegexNameable("^(.*)background_script$") task.OptionalField(bgNameable, schema.TodoSchema, func(node *node.Node) error { command, err := handleBackgroundScript(node, bgNameable) if err != nil { @@ -99,7 +99,7 @@ func NewTask(env map[string]string) *Task { return nil }) - scriptNameable := nameable.NewRegexNameable("(.*)script") + scriptNameable := nameable.NewRegexNameable("^(.*)script$") task.OptionalField(scriptNameable, schema.TodoSchema, func(node *node.Node) error { command, err := handleScript(node, scriptNameable) if err != nil { @@ -111,7 +111,7 @@ func NewTask(env map[string]string) *Task { return nil }) - cacheNameable := nameable.NewRegexNameable("(.*)cache") + cacheNameable := nameable.NewRegexNameable("^(.*)cache$") task.OptionalField(cacheNameable, schema.TodoSchema, func(node *node.Node) error { cache := NewCacheCommand(environment.Merge(task.proto.Environment, env)) if err := cache.Parse(node); err != nil { @@ -149,6 +149,17 @@ func NewTask(env map[string]string) *Task { return nil }) + task.CollectibleField("timeout_in", schema.TodoSchema, func(node *node.Node) error { + timeoutInSeconds, err := handleTimeoutIn(node, environment.Merge(task.proto.Environment, env)) + if err != nil { + return err + } + + task.proto.Metadata.Properties["timeoutInSeconds"] = timeoutInSeconds + + return nil + }) + return task } diff --git a/pkg/parser/testdata/via-rpc/aliases-many.json b/pkg/parser/testdata/via-rpc/aliases-many.json new file mode 100644 index 00000000..c1bb197d --- /dev/null +++ b/pkg/parser/testdata/via-rpc/aliases-many.json @@ -0,0 +1,106 @@ +[ + { + "commands": [ + { + "cloneInstruction": {}, + "name": "clone" + }, + { + "cacheInstruction": { + "fingerprintScripts": [ + "elixir --version", + "cat mix.lock" + ], + "folder": "deps", + "populateScripts": [ + "mix deps.get" + ] + }, + "name": "deps" + }, + { + "cacheInstruction": { + "fingerprintScripts": [ + "elixir --version", + "cat mix.lock" + ], + "folder": "_build", + "populateScripts": [ + "mix deps.compile --long-compilation-threshold 999" + ] + }, + "name": "build" + }, + { + "name": "compile", + "scriptInstruction": { + "scripts": [ + "rm -rf _build/${MIX_ENV}/lib/control_room", + "mix compile --warnings-as-errors --long-compilation-threshold 999" + ] + } + }, + { + "name": "Upload 'deps' cache", + "uploadCacheInstruction": { + "cacheName": "deps" + } + }, + { + "name": "Upload 'build' cache", + "uploadCacheInstruction": { + "cacheName": "build" + } + } + ], + "environment": { + "APP_NAME": "test" + }, + "instance": { + "@type": "type.googleapis.com/org.cirruslabs.ci.services.cirruscigrpc.ContainerInstance", + "cpu": 8, + "image": "alpine:latest", + "memory": 4096 + }, + "metadata": { + "properties": { + "allowFailures": "false", + "executionLock": "null", + "experimentalFeaturesEnabled": "false", + "indexWithinBuild": "0", + "timeoutInSeconds": "1800", + "triggerType": "AUTOMATIC" + } + }, + "name": "compile" + }, + { + "commands": [ + { + "cloneInstruction": {}, + "name": "clone" + } + ], + "environment": { + "APP_NAME": "test" + }, + "instance": { + "@type": "type.googleapis.com/org.cirruslabs.ci.services.cirruscigrpc.ContainerInstance", + "cpu": 2, + "image": "alpine:latest", + "memory": 4096 + }, + "localGroupId": "1", + "metadata": { + "properties": { + "allowFailures": "false", + "executionLock": "null", + "experimentalFeaturesEnabled": "false", + "indexWithinBuild": "1", + "timeoutInSeconds": "1800", + "triggerType": "AUTOMATIC" + } + }, + "name": "main" + } +] diff --git a/pkg/parser/testdata/via-rpc/aliases-many.yml b/pkg/parser/testdata/via-rpc/aliases-many.yml new file mode 100644 index 00000000..755fcbbf --- /dev/null +++ b/pkg/parser/testdata/via-rpc/aliases-many.yml @@ -0,0 +1,136 @@ +container: + image: alpine:latest + +base: &base + timeout_in: 30m + env: + APP_NAME: test + +cache: &cache + deps_cache: + folder: deps + fingerprint_script: + - elixir --version + - cat mix.lock + populate_script: mix deps.get + build_cache: + folder: _build + fingerprint_script: + - elixir --version + - cat mix.lock + populate_script: mix deps.compile --long-compilation-threshold 999 + +node_modules_cache: &node_modules_cache + node_modules_cache: + folder: node_modules + fingerprint_script: cat yarn.lock + populate_script: + - yarn install --production=false + +compile_task: + <<: *base + <<: *cache + name: compile + container: + cpu: 8 + compile_script: + - rm -rf _build/${MIX_ENV}/lib/control_room + - mix compile --warnings-as-errors --long-compilation-threshold 999 + +base1_task_base: &base1 + <<: *base + <<: *cache + <<: *node_modules_cache + +base2_task_base: &base2 + <<: *base + <<: *cache + <<: *node_modules_cache + +base3_task_base: &base3 + <<: *base + <<: *cache + <<: *node_modules_cache + +base4_task_base: &base4 + <<: *base + <<: *cache + <<: *node_modules_cache + +base5_task_base: &base5 + <<: *base + <<: *cache + <<: *node_modules_cache + +base6_task_base: &base6 + <<: *base + <<: *cache + <<: *node_modules_cache + +base7_task_base: &base7 + <<: *base + <<: *cache + <<: *node_modules_cache + +base8_task_base: &base8 + <<: *base + <<: *cache + <<: *node_modules_cache + +base9_task_base: &base9 + <<: *base + <<: *cache + <<: *node_modules_cache + +base10_task_base: &base10 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base11_task_base: &base11 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base12_task_base: &base12 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base13_task_base: &base13 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base14_task_base: &base14 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base15_task_base: &base15 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base16_task_base: &base16 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base17_task_base: &base17 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base18_task_base: &base18 + <<: *base1 + <<: *cache + <<: *node_modules_cache + +base19_task_base: &base19 + <<: *base + <<: *cache + <<: *node_modules_cache + +task: + <<: *base