More RPC-parser-powered tests: simple (#119)

* Additional containers: support port mapping * Additional containers: support command, readiness_command and privileged * Remove github.com/goccy/go-yaml remnants * Make tasks's "allow_failures" field collectible Note that this doesn't apply to the same field in the pipe parser. * Support task and pipe "experimental" and "trigger_type" fields * When handling scalar scripts, split them into multiple lines * TasksToJSON: suppress deprecated_instance field when generating JSON * Simple tests
cirruslabs · Oct 6, 2020 · 81bd388 · 81bd388
1 parent a9746b2
commit 81bd388
Show file tree

Hide file tree

Showing 33 changed files with 983 additions and 21 deletions.
diff --git a/internal/commands/internal/test.go b/internal/commands/internal/test.go
@@ -7,8 +7,8 @@ import (
 	"github.com/cirruslabs/cirrus-cli/pkg/larker"
 	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/local"
 	"github.com/go-test/deep"
-	"github.com/goccy/go-yaml"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -69,13 +69,13 @@ func test(cmd *cobra.Command, args []string) error {
 
 		// Compare generated configuration with the expected configuration
 		var expectedConfig yaml.MapSlice
-		err = yaml.UnmarshalWithOptions(expectedConfigBytes, &expectedConfig, yaml.UseOrderedMap())
+		err = yaml.Unmarshal(expectedConfigBytes, &expectedConfig)
 		if err != nil {
 			return fmt.Errorf("%w: %v", ErrTest, err)
 		}
 
 		var generatedConfig yaml.MapSlice
-		err = yaml.UnmarshalWithOptions([]byte(generatedConfigString), &generatedConfig, yaml.UseOrderedMap())
+		err = yaml.Unmarshal([]byte(generatedConfigString), &generatedConfig)
 		if err != nil {
 			return fmt.Errorf("%w: %v", ErrTest, err)
 		}

diff --git a/internal/testutil/api.go b/internal/testutil/api.go
@@ -13,6 +13,9 @@ func TasksToJSON(t *testing.T, tasks []*api.Task) []byte {
 	for _, task := range tasks {
 		var unmarshalledTask interface{}
 
+		// Shun obsolete fields
+		task.DeprecatedInstance = nil
+
 		marshalled, err := protojson.MarshalOptions{Indent: "  "}.Marshal(task)
 		if err != nil {
 			t.Fatal(err)

diff --git a/pkg/parser/instance/additionalcontainer.go b/pkg/parser/instance/additionalcontainer.go
@@ -25,6 +25,7 @@ type AdditionalContainer struct {
 	parseable.DefaultParser
 }
 
+// nolint:gocognit
 func NewAdditionalContainer(mergedEnv map[string]string) *AdditionalContainer {
 	ac := &AdditionalContainer{
 		proto: &api.AdditionalContainer{},
@@ -86,11 +87,33 @@ func NewAdditionalContainer(mergedEnv map[string]string) *AdditionalContainer {
 		if err != nil {
 			return err
 		}
-		parsedPort, err := strconv.ParseUint(port, 10, 32)
-		if err != nil {
-			return err
+
+		// Support port mapping where a host port[1] is specified in addition to container port
+		// [1]: https://cirrus-ci.org/guide/writing-tasks/#additional-containers
+		const maxSplits = 2
+
+		portParts := strings.SplitN(port, ":", maxSplits)
+
+		if len(portParts) == maxSplits {
+			hostPort, err := strconv.ParseUint(portParts[0], 10, 32)
+			if err != nil {
+				return err
+			}
+			ac.proto.HostPort = uint32(hostPort)
+
+			containerPort, err := strconv.ParseUint(portParts[1], 10, 32)
+			if err != nil {
+				return err
+			}
+			ac.proto.ContainerPort = uint32(containerPort)
+		} else {
+			containerPort, err := strconv.ParseUint(portParts[0], 10, 32)
+			if err != nil {
+				return err
+			}
+			ac.proto.ContainerPort = uint32(containerPort)
 		}
-		ac.proto.ContainerPort = uint32(parsedPort)
+
 		return nil
 	})
 
@@ -120,6 +143,39 @@ func NewAdditionalContainer(mergedEnv map[string]string) *AdditionalContainer {
 		return nil
 	})
 
+	ac.OptionalField(nameable.NewSimpleNameable("command"), schema.TodoSchema, func(node *node.Node) error {
+		command, err := node.GetSliceOfNonEmptyStrings()
+		if err != nil {
+			return err
+		}
+
+		ac.proto.Command = command
+
+		return nil
+	})
+
+	ac.OptionalField(nameable.NewSimpleNameable("readiness_command"), schema.TodoSchema, func(node *node.Node) error {
+		readinessCommand, err := node.GetSliceOfNonEmptyStrings()
+		if err != nil {
+			return err
+		}
+
+		ac.proto.ReadinessCommand = readinessCommand
+
+		return nil
+	})
+
+	ac.OptionalField(nameable.NewSimpleNameable("privileged"), schema.TodoSchema, func(node *node.Node) error {
+		privileged, err := node.GetBoolValue(mergedEnv)
+		if err != nil {
+			return err
+		}
+
+		ac.proto.Privileged = privileged
+
+		return nil
+	})
+
 	return ac
 }
 

diff --git a/pkg/parser/node/accessor.go b/pkg/parser/node/accessor.go
@@ -106,6 +106,17 @@ func (node *Node) GetSliceOfNonEmptyStrings() ([]string, error) {
 	}
 }
 
+func (node *Node) GetScript() ([]string, error) {
+	switch value := node.Value.(type) {
+	case *ScalarValue:
+		return strings.Split(value.Value, "\n"), nil
+	case *ListValue:
+		return node.GetSliceOfStrings()
+	default:
+		return nil, fmt.Errorf("%w: field should be a string or a list of values", parsererror.ErrParsing)
+	}
+}
+
 func ExpandEnvironmentVariables(s string, env map[string]string) string {
 	const maxExpansionIterations = 10
 

diff --git a/pkg/parser/task/command.go b/pkg/parser/task/command.go
@@ -45,7 +45,7 @@ func NewCacheCommand(mergedEnv map[string]string) *CacheCommand {
 	})
 
 	cache.OptionalField(nameable.NewSimpleNameable("fingerprint_script"), schema.TodoSchema, func(node *node.Node) error {
-		scripts, err := node.GetSliceOfNonEmptyStrings()
+		scripts, err := node.GetScript()
 		if err != nil {
 			return err
 		}
@@ -62,7 +62,7 @@ func NewCacheCommand(mergedEnv map[string]string) *CacheCommand {
 	})
 
 	cache.OptionalField(nameable.NewSimpleNameable("populate_script"), schema.TodoSchema, func(node *node.Node) error {
-		scripts, err := node.GetSliceOfNonEmptyStrings()
+		scripts, err := node.GetScript()
 		if err != nil {
 			return err
 		}

diff --git a/pkg/parser/task/handler.go b/pkg/parser/task/handler.go
@@ -8,7 +8,7 @@ import (
 )
 
 func handleBackgroundScript(node *node.Node, nameable *nameable.RegexNameable) (*api.Command, error) {
-	scripts, err := node.GetSliceOfNonEmptyStrings()
+	scripts, err := node.GetScript()
 	if err != nil {
 		return nil, err
 	}
@@ -24,7 +24,7 @@ func handleBackgroundScript(node *node.Node, nameable *nameable.RegexNameable) (
 }
 
 func handleScript(node *node.Node, nameable *nameable.RegexNameable) (*api.Command, error) {
-	scripts, err := node.GetSliceOfNonEmptyStrings()
+	scripts, err := node.GetScript()
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/parser/task/pipe.go b/pkg/parser/task/pipe.go
@@ -12,6 +12,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/schema"
 	"github.com/golang/protobuf/ptypes"
 	"strconv"
+	"strings"
 )
 
 const (
@@ -115,6 +116,15 @@ func NewDockerPipe(env map[string]string) *DockerPipe {
 		return nil
 	})
 
+	pipe.OptionalField(nameable.NewSimpleNameable("experimental"), schema.TodoSchema, func(node *node.Node) error {
+		evaluation, err := node.GetBoolValue(environment.Merge(pipe.proto.Environment, env))
+		if err != nil {
+			return err
+		}
+		pipe.proto.Metadata.Properties["experimentalFeaturesEnabled"] = strconv.FormatBool(evaluation)
+		return nil
+	})
+
 	pipe.CollectibleField("timeout_in", schema.TodoSchema, func(node *node.Node) error {
 		timeoutInSeconds, err := handleTimeoutIn(node, environment.Merge(pipe.proto.Environment, env))
 		if err != nil {
@@ -126,6 +136,15 @@ func NewDockerPipe(env map[string]string) *DockerPipe {
 		return nil
 	})
 
+	pipe.CollectibleField("trigger_type", schema.TodoSchema, func(node *node.Node) error {
+		triggerType, err := node.GetExpandedStringValue(environment.Merge(pipe.proto.Environment, env))
+		if err != nil {
+			return err
+		}
+		pipe.proto.Metadata.Properties["triggerType"] = strings.ToUpper(triggerType)
+		return nil
+	})
+
 	return pipe
 }
 

diff --git a/pkg/parser/task/pipestep.go b/pkg/parser/task/pipestep.go
@@ -34,18 +34,13 @@ func NewPipeStep(mergedEnv map[string]string) *PipeStep {
 
 	scriptNameable := nameable.NewRegexNameable("^(.*)script$")
 	step.OptionalField(scriptNameable, schema.TodoSchema, func(node *node.Node) error {
-		scripts, err := node.GetSliceOfNonEmptyStrings()
+		command, err := handleScript(node, scriptNameable)
 		if err != nil {
 			return err
 		}
-		step.protoCommands = append(step.protoCommands, &api.Command{
-			Name: scriptNameable.FirstGroupOrDefault(node.Name, "main"),
-			Instruction: &api.Command_ScriptInstruction{
-				ScriptInstruction: &api.ScriptInstruction{
-					Scripts: scripts,
-				},
-			},
-		})
+
+		step.protoCommands = append(step.protoCommands, command)
+
 		return nil
 	})
 

diff --git a/pkg/parser/task/task.go b/pkg/parser/task/task.go
@@ -171,7 +171,8 @@ func NewTask(env map[string]string, additionalInstances map[string]protoreflect.
 		task.onlyIfExpression = onlyIfExpression
 		return nil
 	})
-	task.OptionalField(nameable.NewSimpleNameable("allow_failures"), schema.TodoSchema, func(node *node.Node) error {
+
+	task.CollectibleField("allow_failures", schema.TodoSchema, func(node *node.Node) error {
 		evaluation, err := node.GetBoolValue(environment.Merge(task.proto.Environment, env))
 		if err != nil {
 			return err
@@ -180,6 +181,15 @@ func NewTask(env map[string]string, additionalInstances map[string]protoreflect.
 		return nil
 	})
 
+	task.CollectibleField("experimental", schema.TodoSchema, func(node *node.Node) error {
+		evaluation, err := node.GetBoolValue(environment.Merge(task.proto.Environment, env))
+		if err != nil {
+			return err
+		}
+		task.proto.Metadata.Properties["experimentalFeaturesEnabled"] = strconv.FormatBool(evaluation)
+		return nil
+	})
+
 	task.CollectibleField("timeout_in", schema.TodoSchema, func(node *node.Node) error {
 		timeoutInSeconds, err := handleTimeoutIn(node, environment.Merge(task.proto.Environment, env))
 		if err != nil {
@@ -191,6 +201,15 @@ func NewTask(env map[string]string, additionalInstances map[string]protoreflect.
 		return nil
 	})
 
+	task.CollectibleField("trigger_type", schema.TodoSchema, func(node *node.Node) error {
+		triggerType, err := node.GetExpandedStringValue(environment.Merge(task.proto.Environment, env))
+		if err != nil {
+			return err
+		}
+		task.proto.Metadata.Properties["triggerType"] = strings.ToUpper(triggerType)
+		return nil
+	})
+
 	task.OptionalField(nameable.NewSimpleNameable("execution_lock"), schema.TodoSchema, func(node *node.Node) error {
 		lockName, err := node.GetExpandedStringValue(environment.Merge(task.proto.Environment, env))
 		if err != nil {

diff --git a/pkg/parser/testdata/via-rpc/simple-additionalContainers.json b/pkg/parser/testdata/via-rpc/simple-additionalContainers.json
@@ -0,0 +1,69 @@
+[
+  {
+    "commands": [
+      {
+        "cloneInstruction": {},
+        "name": "clone"
+      },
+      {
+        "name": "main",
+        "scriptInstruction": {
+          "scripts": [
+            "./test1.sh"
+          ]
+        }
+      }
+    ],
+    "instance": {
+      "@type": "type.googleapis.com/org.cirruslabs.ci.services.cirruscigrpc.ContainerInstance",
+      "additionalContainers": [
+        {
+          "command": [
+            "echo $FOO"
+          ],
+          "containerPort": 6379,
+          "cpu": 1.3,
+          "environment": {
+            "FOO": "Bar"
+          },
+          "image": "memcached:1.5.0-alpine",
+          "memory": 777,
+          "name": "memcached",
+          "readinessCommand": [
+            "./health.sh"
+          ]
+        },
+        {
+          "containerPort": 6379,
+          "cpu": 0.5,
+          "hostPort": 7777,
+          "image": "redis:3.2-alpine",
+          "memory": 512,
+          "name": "redis"
+        },
+        {
+          "containerPort": 2375,
+          "cpu": 0.5,
+          "image": "docker:18.06-dind",
+          "memory": 512,
+          "name": "docker",
+          "privileged": true
+        }
+      ],
+      "cpu": 2,
+      "image": "my-service:latest",
+      "memory": 4096
+    },
+    "metadata": {
+      "properties": {
+        "allowFailures": "false",
+        "executionLock": "null",
+        "experimentalFeaturesEnabled": "false",
+        "indexWithinBuild": "0",
+        "timeoutInSeconds": "3600",
+        "triggerType": "AUTOMATIC"
+      }
+    },
+    "name": "task1"
+  }
+]
diff --git a/pkg/parser/testdata/via-rpc/simple-additionalContainers.yml b/pkg/parser/testdata/via-rpc/simple-additionalContainers.yml
@@ -0,0 +1,23 @@
+container:
+  image: my-service:latest
+  additional_containers:
+    - name: memcached
+      image: memcached:1.5.0-alpine
+      port: 6379
+      cpu: 1.3
+      memory: 777
+      command: echo $FOO
+      readiness_command: ./health.sh
+      environment:
+        FOO: Bar
+    - name: redis
+      image: redis:3.2-alpine
+      port: 7777:6379
+    - name: docker
+      image: docker:18.06-dind
+      port: 2375
+      privileged: true
+
+task:
+  name: task1
+  script: ./test1.sh