simplify Evaluable interface and Scenario.Run

No longer pass the `*testing.T` down into Plugin's Spec.Eval() method,
freeing up plugins to focus on actions and assertions. Scenario.Run()
now handles the `*testing.T` and the trace context centrally instead of
plugins needing to do this.

Also simplifies the Scenario.Run() interface and Evaluable.Eval()
interfaces to return (result.Result, error) to more cleanly pass back
RuntimeErrors instead of needing a Result.RuntimeError() method.

Signed-off-by: Jay Pipes <[email protected]>

README.md

@@ -540,6 +540,108 @@ test spec also contains these fields:
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26
 
+### Timeouts and retrying assertions
+
+When evaluating assertions for a test spec, `gdt` inspects the test's
+`timeout` value to determine how long to retry the `get` call and recheck
+the assertions.
+
+If a test's `timeout` is empty, `gdt` inspects the scenario's
+`defaults.timeout` value. If both of those values are empty, `gdt` will look
+for any default `timeout` value that the plugin uses.
+
+If you're interested in seeing the individual results of `gdt`'s
+assertion-checks for a single `get` call, you can use the `gdt.WithDebug()`
+function, like this test function demonstrates:
+
+file: `testdata/matches.yaml`:
+
+```yaml
+name: matches
+description: create a deployment and check the matches condition succeeds
+fixtures:
+  - kind
+tests:
+  - name: create-deployment
+    kube:
+      create: testdata/manifests/nginx-deployment.yaml
+  - name: deployment-exists
+    kube:
+      get: deployments/nginx
+    assert:
+      matches:
+        spec:
+          replicas: 2
+          template:
+            metadata:
+              labels:
+                app: nginx
+        status:
+          readyReplicas: 2
+  - name: delete-deployment
+    kube:
+      delete: deployments/nginx
+```
+
+file: `matches_test.go`
+
+```go
+import (
+    "github.com/gdt-dev/gdt"
+    _ "github.com/gdt-dev/kube"
+    kindfix "github.com/gdt-dev/kube/fixture/kind"
+)
+
+func TestMatches(t *testing.T) {
+	fp := filepath.Join("testdata", "matches.yaml")
+
+	kfix := kindfix.New()
+
+	s, err := gdt.From(fp)
+
+	ctx := gdt.NewContext(gdt.WithDebug())
+	ctx = gdt.RegisterFixture(ctx, "kind", kfix)
+	s.Run(ctx, t)
+}
+```
+
+Here's what running `go test -v matches_test.go` would look like:
+
+```
+$ go test -v matches_test.go
+=== RUN   TestMatches
+=== RUN   TestMatches/matches
+=== RUN   TestMatches/matches/create-deployment
+=== RUN   TestMatches/matches/deployment-exists
+deployment-exists (try 1 after 1.303µs) ok: false, terminal: false
+deployment-exists (try 1 after 1.303µs) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 2 after 595.62786ms) ok: false, terminal: false
+deployment-exists (try 2 after 595.62786ms) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 3 after 1.020003807s) ok: false, terminal: false
+deployment-exists (try 3 after 1.020003807s) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 4 after 1.760006109s) ok: false, terminal: false
+deployment-exists (try 4 after 1.760006109s) failure: assertion failed: match field not equal: $.status.readyReplicas had different values. expected 2 but found 1
+deployment-exists (try 5 after 2.772416449s) ok: true, terminal: false
+=== RUN   TestMatches/matches/delete-deployment
+--- PASS: TestMatches (3.32s)
+    --- PASS: TestMatches/matches (3.30s)
+        --- PASS: TestMatches/matches/create-deployment (0.01s)
+        --- PASS: TestMatches/matches/deployment-exists (2.78s)
+        --- PASS: TestMatches/matches/delete-deployment (0.02s)
+PASS
+ok  	command-line-arguments	3.683s
+```
+
+You can see from the debug output above that `gdt` created the Deployment and
+then did a `kube.get` for the `deployments/nginx` Deployment. Initially
+(attempt 1), the `assert.matches` assertion failed because the
+`status.readyReplicas` field was not present in the returned resource. `gdt`
+retried the `kube.get` call 4 more times (attempts 2-5), with attempts 2 and 3
+failed the existence check for the `status.readyReplicas` field and attempt 4
+failing the *value* check for the `status.readyReplicas` field being `1`
+instead of the expected `2`. Finally, when the Deployment was completely rolled
+out, attempt 5 succeeded in all the `assert.matches` assertions.
+
 ## Contributing and acknowledgements
 
 `gdt` was inspired by [Gabbi](https://github.com/cdent/gabbi), the excellent

context/context_test.go

@@ -43,8 +43,8 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 type fooPlugin struct{}

plugin/exec/action.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"context"
 	"os/exec"
-	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
@@ -38,7 +37,6 @@ type Action struct {
 // respectively.
 func (a *Action) Do(
 	ctx context.Context,
-	t *testing.T,
 	outbuf *bytes.Buffer,
 	errbuf *bytes.Buffer,
 	exitcode *int,

plugin/exec/eval.go

@@ -7,7 +7,6 @@ package exec
 import (
 	"bytes"
 	"context"
-	"testing"
 
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
@@ -17,30 +16,34 @@ import (
 // Eval performs an action and evaluates the results of that action, returning
 // a Result that informs the Scenario about what failed or succeeded about the
 // Evaluable's conditions.
-func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
+//
+// Errors returned by Eval() are **RuntimeErrors**, not failures in assertions.
+func (s *Spec) Eval(
+	ctx context.Context,
+) (*result.Result, error) {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
 	var ec int
 
-	if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
+	if err := s.Do(ctx, outbuf, errbuf, &ec); err != nil {
 		if err == gdterrors.ErrTimeoutExceeded {
-			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
+			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)), nil
 		}
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
+		return nil, ExecRuntimeError(err)
 	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK(ctx) {
 		if s.On != nil {
 			if s.On.Fail != nil {
 				outbuf.Reset()
 				errbuf.Reset()
-				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
+				err := s.On.Fail.Do(ctx, outbuf, errbuf, nil)
 				if err != nil {
 					debug.Println(ctx, "error in on.fail.exec: %s", err)
 				}
 			}
 		}
 	}
-	return result.New(result.WithFailures(a.Failures()...))
+	return result.New(result.WithFailures(a.Failures()...)), nil
 }

plugin/registry_test.go

@@ -36,8 +36,8 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Eval(context.Context, *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {

result/result.go

@@ -4,13 +4,6 @@
 
 package result
 
-import (
-	"errors"
-	"fmt"
-
-	gdterrors "github.com/gdt-dev/gdt/errors"
-)
-
 // Result is returned from a `Evaluable.Eval` execution. It serves two
 // purposes:
 //
@@ -23,9 +16,6 @@ import (
 // returned in the Result and the `Scenario.Run` method injects that
 // information into the context that is supplied to the next Spec's `Run`.
 type Result struct {
-	// err is any error that was returned from the Evaluable's execution. This
-	// is guaranteed to be a `gdterrors.RuntimeError`.
-	err error
 	// failures is the collection of error messages from assertion failures
 	// that occurred during Eval(). These are *not* `gdterrors.RuntimeError`.
 	failures []error
@@ -36,17 +26,6 @@ type Result struct {
 	data map[string]interface{}
 }
 
-// HasRuntimeError returns true if the Eval() returned a runtime error, false
-// otherwise.
-func (r *Result) HasRuntimeError() bool {
-	return r.err != nil
-}
-
-// RuntimeError returns the runtime error
-func (r *Result) RuntimeError() error {
-	return r.err
-}
-
 // HasData returns true if any of the run data has been set, false otherwise.
 func (r *Result) HasData() bool {
 	return r.data != nil
@@ -86,19 +65,6 @@ func (r *Result) SetFailures(failures ...error) {
 
 type ResultModifier func(*Result)
 
-// WithRuntimeError modifies the Result with the supplied error
-func WithRuntimeError(err error) ResultModifier {
-	if !errors.Is(err, gdterrors.RuntimeError) {
-		msg := fmt.Sprintf("expected %s to be a gdterrors.RuntimeError", err)
-		// panic here because a plugin author incorrectly implemented their
-		// plugin Spec's Eval() method...
-		panic(msg)
-	}
-	return func(r *Result) {
-		r.err = err
-	}
-}
-
 // WithData modifies the Result with the supplied run data key and value
 func WithData(key string, val interface{}) ResultModifier {
 	return func(r *Result) {