feat(predicate): Add has_workflow_result predicate

This is a new predicate which matches on the results of an entire GitHub
Actions workflow. This is similar to `has_status`, except this matches
on statuses only, which for GitHub Actions are roughly equivalent to
_jobs_ in a Workflow.

The workflows are given as paths in the repository, as these can't be
dynamic and are easy to predict when writing policies.

README.md

@@ -329,6 +329,16 @@ if:
     - "status-name-2"
     - "status-name-3"
 
+  # "has_workflow_result" is satisfied if the GitHub Actions workflow runs that
+  # are specified all finished and concluded with one of the conclusions
+  # specified. "conclusions" is optional and defaults to ["success"].
+  # `workflows` contains the paths to the workflow files that are being checked.
+  has_workflow_result:
+    conclusions: ["success", "skipped"]
+    workflows:
+      - ".github/workflows/a.yml"
+      - ".github/workflows/b.yml"
+
   # "has_labels" is satisfied if the pull request has the specified labels
   # applied
   has_labels:
@@ -983,6 +993,7 @@ The app requires these permissions:
 
 | Permission | Access | Reason |
 | ---------- | ------ | ------ |
+| Actions| Read-only | Read workflow run events for the `has_workflow_result` predicate |
 | Repository contents | Read-only | Read configuration and commit metadata |
 | Checks | Read-only | Read check run results |
 | Repository administration | Read-only | Read admin team(s) membership |
@@ -1001,6 +1012,7 @@ The app should be subscribed to these events:
 * Pull request
 * Pull request review
 * Status
+* Workflow Run
 
 There is a [`logo.png`](https://github.com/palantir/policy-bot/blob/develop/logo.png)
 provided if you'd like to use it as the GitHub application logo. The background

policy/predicate/predicates.go

@@ -31,6 +31,8 @@ type Predicates struct {
 
 	HasSuccessfulStatus *HasSuccessfulStatus `yaml:"has_successful_status"`
 
+	HasWorkflowResult *HasWorkflowResult `yaml:"has_workflow_result"`
+
 	HasLabels *HasLabels `yaml:"has_labels"`
 
 	Repository *Repository `yaml:"repository"`
@@ -82,6 +84,10 @@ func (p *Predicates) Predicates() []Predicate {
 		ps = append(ps, Predicate(p.HasSuccessfulStatus))
 	}
 
+	if p.HasWorkflowResult != nil {
+		ps = append(ps, Predicate(p.HasWorkflowResult))
+	}
+
 	if p.HasLabels != nil {
 		ps = append(ps, Predicate(p.HasLabels))
 	}

policy/predicate/workflow.go

@@ -0,0 +1,95 @@
+// Copyright 2018 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package predicate
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/palantir/policy-bot/policy/common"
+	"github.com/palantir/policy-bot/pull"
+	"github.com/pkg/errors"
+)
+
+type HasWorkflowResult struct {
+	Conclusions allowedConclusions `yaml:"conclusions"`
+	Workflows   []string           `yaml:"workflows"`
+}
+
+func NewHasWorkflowResult(workflows []string, conclusions []string) *HasWorkflowResult {
+	conclusionsSet := make(allowedConclusions, len(conclusions))
+	for _, conclusion := range conclusions {
+		conclusionsSet[conclusion] = unit{}
+	}
+	return &HasWorkflowResult{
+		Conclusions: conclusionsSet,
+		Workflows:   workflows,
+	}
+}
+
+var _ Predicate = HasWorkflowResult{}
+
+func (pred HasWorkflowResult) Evaluate(ctx context.Context, prctx pull.Context) (*common.PredicateResult, error) {
+	workflowRuns, err := prctx.LatestWorkflowRuns()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list latest workflow runs")
+	}
+
+	conclusions := pred.Conclusions
+	if len(conclusions) == 0 {
+		conclusions = defaultConclusions
+	}
+
+	predicateResult := common.PredicateResult{
+		ValuePhrase:     "workflow results",
+		ConditionPhrase: fmt.Sprintf("exist and have conclusion %s", conclusions.joinWithOr()),
+	}
+
+	var missingResults []string
+	var failingWorkflows []string
+	for _, workflow := range pred.Workflows {
+		result, ok := workflowRuns[workflow]
+		if !ok {
+			missingResults = append(missingResults, workflow)
+		}
+		if _, allowed := conclusions[result]; !allowed {
+			failingWorkflows = append(failingWorkflows, workflow)
+		}
+	}
+
+	if len(missingResults) > 0 {
+		predicateResult.Values = missingResults
+		predicateResult.Description = "One or more workflow runs are  missing: " + strings.Join(missingResults, ", ")
+		predicateResult.Satisfied = false
+		return &predicateResult, nil
+	}
+
+	if len(failingWorkflows) > 0 {
+		predicateResult.Values = failingWorkflows
+		predicateResult.Description = fmt.Sprintf("One or more workflow runs have not concluded with %s: %s", pred.Conclusions.joinWithOr(), strings.Join(failingWorkflows, ","))
+		predicateResult.Satisfied = false
+		return &predicateResult, nil
+	}
+
+	predicateResult.Values = pred.Workflows
+	predicateResult.Satisfied = true
+
+	return &predicateResult, nil
+}
+
+func (pred HasWorkflowResult) Trigger() common.Trigger {
+	return common.TriggerStatus
+}

policy/predicate/workflow_test.go

@@ -0,0 +1,217 @@
+// Copyright 2019 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package predicate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/palantir/policy-bot/policy/common"
+	"github.com/palantir/policy-bot/pull/pulltest"
+	"github.com/stretchr/testify/assert"
+)
+
+type WorkflowTestCase struct {
+	name                    string
+	latestWorkflowRunsValue map[string]string
+	latestWorkflowRunsError error
+	predicate               Predicate
+	ExpectedPredicateResult *common.PredicateResult
+}
+
+func TestHasSuccessfulWorkflowRun(t *testing.T) {
+	commonTestCases := []WorkflowTestCase{
+		{
+			name: "all workflows succeed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml": "success",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: true,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name: "multiple workflows succeed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "success",
+				".github/workflows/test2.yml": "success",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: true,
+				Values:    []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+		},
+		{
+			name: "a workflow fails",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml": "failure",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name: "multiple workflows fail",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "failure",
+				".github/workflows/test2.yml": "failure",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+		},
+		{
+			name: "one success, one failure",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "success",
+				".github/workflows/test2.yml": "failure",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test2.yml"},
+			},
+		},
+		{
+			name:                    "a workflow is missing",
+			latestWorkflowRunsValue: map[string]string{},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name:                    "multiple workflow are missing",
+			latestWorkflowRunsValue: map[string]string{},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+		},
+		{
+			name: "a workflow is missing, the other workflow is skipped",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test2.yml": "skipped",
+			},
+			predicate: HasWorkflowResult{
+				Workflows: []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name: "a workflow is skipped, but skipped workflows are allowed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml": "skipped",
+			},
+			predicate: HasWorkflowResult{
+				Workflows:   []string{".github/workflows/test.yml"},
+				Conclusions: allowedConclusions{"skipped": {}},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: true,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name: "a workflow succeeds, the other workflow is skipped, but skipped workflows are allowed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "success",
+				".github/workflows/test2.yml": "skipped",
+			},
+			predicate: HasWorkflowResult{
+				Workflows:   []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+				Conclusions: allowedConclusions{"skipped": {}, "success": {}},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: true,
+				Values:    []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+			},
+		},
+		{
+			name: "a workflow fails, the other workflow is skipped, but skipped workflows are allowed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "failure",
+				".github/workflows/test2.yml": "skipped",
+			},
+			predicate: HasWorkflowResult{
+				Workflows:   []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+				Conclusions: allowedConclusions{"skipped": {}, "success": {}},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+		{
+			name: "a workflow succeeds, the other workflow is skipped, only skipped workflows are allowed",
+			latestWorkflowRunsValue: map[string]string{
+				".github/workflows/test.yml":  "success",
+				".github/workflows/test2.yml": "skipped",
+			},
+			predicate: HasWorkflowResult{
+				Workflows:   []string{".github/workflows/test.yml", ".github/workflows/test2.yml"},
+				Conclusions: allowedConclusions{"skipped": {}},
+			},
+			ExpectedPredicateResult: &common.PredicateResult{
+				Satisfied: false,
+				Values:    []string{".github/workflows/test.yml"},
+			},
+		},
+	}
+
+	runWorkflowTestCase(t, commonTestCases)
+}
+
+func runWorkflowTestCase(t *testing.T, cases []WorkflowTestCase) {
+	ctx := context.Background()
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			predicateResult, err := tc.predicate.Evaluate(ctx, &pulltest.Context{
+				LatestWorkflowRunsValue: tc.latestWorkflowRunsValue,
+				LatestStatusesError:     tc.latestWorkflowRunsError,
+			})
+			if assert.NoError(t, err, "evaluation failed") {
+				assertPredicateResult(t, tc.ExpectedPredicateResult, predicateResult)
+			}
+		})
+	}
+}

pull/context.go

@@ -124,6 +124,11 @@ type Context interface {
 	// LatestStatuses returns a map of status check names to the latest result
 	LatestStatuses() (map[string]string, error)
 
+	// LatestWorkflowRuns returns the latest GitHub Actions workflow runs for
+	// the pull request. The keys of the map are paths to the workflow files and
+	// the values are the conclusion of the latest run.
+	LatestWorkflowRuns() (map[string]string, error)
+
 	// Labels returns a list of labels applied on the Pull Request
 	Labels() ([]string, error)
 }