Merge pull request #1858 from lcarva/EC-682

fix: allow validate input to handle multiple source groups

acceptance/examples/ham.rego

@@ -0,0 +1,18 @@
+package ham
+
+import rego.v1
+
+# METADATA
+# title: Ham
+# description: Ham ham ham
+# custom:
+#   short_name: delicious
+#
+deny contains result if {
+	value := object.get(input, "ham", "yucky")
+	value != "delicious"
+	result := {
+		"msg": "ham is not delicious",
+		"code": "ham.delicious",
+	}
+}

acceptance/examples/multiple_sources_config.yaml

@@ -0,0 +1,6 @@
+---
+sources:
+  - policy:
+      - "git::https://${GITHOST}/git/ham-policy"
+  - policy:
+      - "git::https://${GITHOST}/git/spam-policy"

acceptance/examples/spam.rego

@@ -0,0 +1,18 @@
+package spam
+
+import rego.v1
+
+# METADATA
+# title: Spam
+# description: Spam spam spam
+# custom:
+#   short_name: valid
+#
+deny contains result if {
+	value := object.get(input, "spam", false)
+	not value
+	result := {
+		"msg": "spam is not true",
+		"code": "spam.valid",
+	}
+}

features/__snapshots__/validate_input.snap

@@ -35,8 +35,58 @@
 
 [policy URL with no rego files:stderr - 1]
 Error: 1 error occurred:
-    * error validating file pipeline_definition.yaml: no rego files found in policy subdirectory
+    * error validating file pipeline_definition.yaml: evaluating policy: no rego files found in policy subdirectory
 
 
 
+---
+
+[policy with multiple sources:stdout - 1]
+{
+  "success": false,
+  "filepaths": [
+    {
+      "filepath": "input.yaml",
+      "violations": [
+        {
+          "msg": "ham is not delicious",
+          "metadata": {
+            "code": "ham.delicious"
+          }
+        },
+        {
+          "msg": "spam is not true",
+          "metadata": {
+            "code": "spam.valid"
+          }
+        }
+      ],
+      "warnings": [],
+      "successes": null,
+      "success": false,
+      "success-count": 0
+    }
+  ],
+  "policy": {
+    "sources": [
+      {
+        "policy": [
+          "git::https://${GITHOST}/git/ham-policy"
+        ]
+      },
+      {
+        "policy": [
+          "git::https://${GITHOST}/git/spam-policy"
+        ]
+      }
+    ]
+  },
+  "ec-version": "${EC_VERSION}",
+  "effective-time": "${TIMESTAMP}"
+}
+---
+
+[policy with multiple sources:stderr - 1]
+Error: success criteria not met
+
 ---

features/validate_input.feature

@@ -1,6 +1,6 @@
 Feature: validate input
   The ec command line should be able to inspect input files
-  
+
   Background:
     Given stub git daemon running
 
@@ -49,3 +49,20 @@ Feature: validate input
     When ec command is run with "validate input --file pipeline_definition.yaml --policy git::https://${GITHOST}/git/sad-day-config.git"
     Then the exit status should be 1
     Then the output should match the snapshot
+
+  Scenario: policy with multiple sources
+    Given a git repository named "multiple-sources-config" with
+      | policy.yaml | examples/multiple_sources_config.yaml |
+    Given a git repository named "spam-policy" with
+      | main.rego | examples/spam.rego |
+    Given a git repository named "ham-policy" with
+      | main.rego | examples/ham.rego |
+    Given a pipeline definition file named "input.yaml" containing
+      """
+      ---
+      spam: false
+      ham: rotten
+      """
+    When ec command is run with "validate input --file input.yaml --policy git::https://${GITHOST}/git/multiple-sources-config.git"
+    Then the exit status should be 1
+    Then the output should match the snapshot

internal/evaluation_target/input/input.go

@@ -30,8 +30,8 @@ var newConftestEvaluator = evaluator.NewConftestEvaluator
 
 // Input represents the structure needed to evaluate a generic file input
 type Input struct {
-	Paths     []string
-	Evaluator evaluator.Evaluator
+	Paths      []string
+	Evaluators []evaluator.Evaluator
 }
 
 // NewInput returns a Input struct with FPath and evaluator ready to use
@@ -59,7 +59,7 @@ func NewInput(ctx context.Context, paths []string, p policy.Policy) (*Input, err
 		}
 
 		log.Debug("Conftest evaluator initialized")
-		i.Evaluator = c
+		i.Evaluators = append(i.Evaluators, c)
 
 	}
 	return i, nil

internal/input/validate.go

@@ -46,14 +46,21 @@ func ValidateInput(ctx context.Context, fpath string, policy policy.Policy, deta
 		return nil, err
 	}
 
-	results, _, err := p.Evaluator.Evaluate(ctx, evaluator.EvaluationTarget{Inputs: inputFiles})
-	if err != nil {
-		log.Debug("Problem running conftest policy check!")
-		return nil, err
+	var allResults []evaluator.Outcome
+	for _, e := range p.Evaluators {
+		results, _, err := e.Evaluate(ctx, evaluator.EvaluationTarget{Inputs: inputFiles})
+		if err != nil {
+			return nil, fmt.Errorf("evaluating policy: %w", err)
+		}
+		allResults = append(allResults, results...)
 	}
 
 	log.Debug("Conftest policy check complete")
-	return &output.Output{PolicyCheck: results, Detailed: detailed}, nil
+
+	out := output.Output{Detailed: detailed}
+	out.SetPolicyCheck(allResults)
+
+	return &out, nil
 }
 
 // detect if a file or directory was passed. if a directory, gather all files in it

internal/input/validate_test.go

@@ -64,13 +64,13 @@ func (e badMockEvaluator) CapabilitiesPath() string {
 
 func mockNewPipelineDefinitionFile(ctx context.Context, fpath []string, policy policy.Policy) (*input.Input, error) {
 	return &input.Input{
-		Evaluator: mockEvaluator{},
+		Evaluators: []evaluator.Evaluator{mockEvaluator{}},
 	}, nil
 }
 
 func badMockNewPipelineDefinitionFile(ctx context.Context, fpath []string, policy policy.Policy) (*input.Input, error) {
 	return &input.Input{
-		Evaluator: badMockEvaluator{},
+		Evaluators: []evaluator.Evaluator{badMockEvaluator{}},
 	}, nil
 }
 
@@ -91,7 +91,7 @@ func Test_ValidatePipeline(t *testing.T) {
 			name:    "validation succeeds",
 			fpath:   validFile,
 			err:     nil,
-			output:  &output.Output{PolicyCheck: []evaluator.Outcome{}},
+			output:  &output.Output{},
 			defFunc: mockNewPipelineDefinitionFile,
 		},
 		{
@@ -111,22 +111,22 @@ func Test_ValidatePipeline(t *testing.T) {
 		{
 			name:    "valid file, but evaluator fails",
 			fpath:   validFile,
-			err:     errors.New("Evaluator error"),
+			err:     fmt.Errorf("evaluating policy: %w", errors.New("Evaluator error")),
 			output:  nil,
 			defFunc: badMockNewPipelineDefinitionFile,
 		},
 		{
 			name:    "validation succeeds with json input",
 			fpath:   "{\"json\": 1}",
 			err:     nil,
-			output:  &output.Output{PolicyCheck: []evaluator.Outcome{}},
+			output:  &output.Output{},
 			defFunc: mockNewPipelineDefinitionFile,
 		},
 		{
 			name:    "validation succeeds with yaml input",
 			fpath:   "kind: task",
 			err:     nil,
-			output:  &output.Output{PolicyCheck: []evaluator.Outcome{}},
+			output:  &output.Output{},
 			defFunc: mockNewPipelineDefinitionFile,
 		},
 		{