Merge pull request #33 from jaypipes/simplify-eval

simplify Evaluable interface and Scenario.Run

Author: jaypipes

README.md

@@ -540,6 +540,108 @@ test spec also contains these fields:
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26
 
+### Timeouts and retrying assertions
+
+When evaluating assertions for a test spec, `gdt` inspects the test's
+`timeout` value to determine how long to retry the `get` call and recheck
+the assertions.
+
+If a test's `timeout` is empty, `gdt` inspects the scenario's
+`defaults.timeout` value. If both of those values are empty, `gdt` will look
+for any default `timeout` value that the plugin uses.
+
+If you're interested in seeing the individual results of `gdt`'s
+assertion-checks for a single `get` call, you can use the `gdt.WithDebug()`
+function, like this test function demonstrates:
+
+file: `testdata/matches.yaml`:
+
+```yaml
+name: matches
+description: create a deployment and check the matches condition succeeds
+fixtures:
+  - kind
+tests:
+  - name: create-deployment
+    kube:
+      create: testdata/manifests/nginx-deployment.yaml
+  - name: deployment-exists
+    kube:
+      get: deployments/nginx
+    assert:
+      matches:
+        spec:
+          replicas: 2
+          template:
+            metadata:
+              labels:
+                app: nginx
+        status:
+          readyReplicas: 2
+  - name: delete-deployment
+    kube:
+      delete: deployments/nginx
+```
+
+file: `matches_test.go`
+
+```go
+import (
+    "github.com/gdt-dev/gdt"
+    _ "github.com/gdt-dev/kube"
+    kindfix "github.com/gdt-dev/kube/fixture/kind"
+)
+
+func TestMatches(t *testing.T) {
+	fp := filepath.Join("testdata", "matches.yaml")
+
+	kfix := kindfix.New()
+
+	s, err := gdt.From(fp)
+
+	ctx := gdt.NewContext(gdt.WithDebug())
+	ctx = gdt.RegisterFixture(ctx, "kind", kfix)
+	s.Run(ctx, t)
+}
+```
+
+Here's what running `go test -v matches_test.go` would look like:
+
+```
+$ go test -v matches_test.go
+=== RUN   TestMatches
+=== RUN   TestMatches/matches
+=== RUN   TestMatches/matches/create-deployment
+=== RUN   TestMatches/matches/deployment-exists
+deployment-exists (try 1 after 1.303µs) ok: false, terminal: false
+deployment-exists (try 1 after 1.303µs) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 2 after 595.62786ms) ok: false, terminal: false
+deployment-exists (try 2 after 595.62786ms) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 3 after 1.020003807s) ok: false, terminal: false
+deployment-exists (try 3 after 1.020003807s) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 4 after 1.760006109s) ok: false, terminal: false
+deployment-exists (try 4 after 1.760006109s) failure: assertion failed: match field not equal: $.status.readyReplicas had different values. expected 2 but found 1
+deployment-exists (try 5 after 2.772416449s) ok: true, terminal: false
+=== RUN   TestMatches/matches/delete-deployment
+--- PASS: TestMatches (3.32s)
+    --- PASS: TestMatches/matches (3.30s)
+        --- PASS: TestMatches/matches/create-deployment (0.01s)
+        --- PASS: TestMatches/matches/deployment-exists (2.78s)
+        --- PASS: TestMatches/matches/delete-deployment (0.02s)
+PASS
+ok  	command-line-arguments	3.683s
+```
+
+You can see from the debug output above that `gdt` created the Deployment and
+then did a `kube.get` for the `deployments/nginx` Deployment. Initially
+(attempt 1), the `assert.matches` assertion failed because the
+`status.readyReplicas` field was not present in the returned resource. `gdt`
+retried the `kube.get` call 4 more times (attempts 2-5), with attempts 2 and 3
+failed the existence check for the `status.readyReplicas` field and attempt 4
+failing the *value* check for the `status.readyReplicas` field being `1`
+instead of the expected `2`. Finally, when the Deployment was completely rolled
+out, attempt 5 succeeded in all the `assert.matches` assertions.
+
 ## Contributing and acknowledgements
 
 `gdt` was inspired by [Gabbi](https://github.com/cdent/gabbi), the excellent

context/context_test.go

@@ -43,8 +43,8 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 type fooPlugin struct{}

plugin/exec/action.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"context"
 	"os/exec"
-	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
@@ -38,7 +37,6 @@ type Action struct {
 // respectively.
 func (a *Action) Do(
 	ctx context.Context,
-	t *testing.T,
 	outbuf *bytes.Buffer,
 	errbuf *bytes.Buffer,
 	exitcode *int,

plugin/exec/eval.go

@@ -7,7 +7,6 @@ package exec
 import (
 	"bytes"
 	"context"
-	"testing"
 
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
@@ -17,30 +16,34 @@ import (
 // Eval performs an action and evaluates the results of that action, returning
 // a Result that informs the Scenario about what failed or succeeded about the
 // Evaluable's conditions.
-func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
+//
+// Errors returned by Eval() are **RuntimeErrors**, not failures in assertions.
+func (s *Spec) Eval(
+	ctx context.Context,
+) (*result.Result, error) {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
 	var ec int
 
-	if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
+	if err := s.Do(ctx, outbuf, errbuf, &ec); err != nil {
 		if err == gdterrors.ErrTimeoutExceeded {
-			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
+			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)), nil
 		}
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
+		return nil, ExecRuntimeError(err)
 	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK(ctx) {
 		if s.On != nil {
 			if s.On.Fail != nil {
 				outbuf.Reset()
 				errbuf.Reset()
-				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
+				err := s.On.Fail.Do(ctx, outbuf, errbuf, nil)
 				if err != nil {
 					debug.Println(ctx, "error in on.fail.exec: %s", err)
 				}
 			}
 		}
 	}
-	return result.New(result.WithFailures(a.Failures()...))
+	return result.New(result.WithFailures(a.Failures()...)), nil
 }

plugin/registry_test.go

@@ -36,8 +36,8 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Eval(context.Context, *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {

result/result.go

@@ -4,13 +4,6 @@
 
 package result
 
-import (
-	"errors"
-	"fmt"
-
-	gdterrors "github.com/gdt-dev/gdt/errors"
-)
-
 // Result is returned from a `Evaluable.Eval` execution. It serves two
 // purposes:
 //
@@ -23,9 +16,6 @@ import (
 // returned in the Result and the `Scenario.Run` method injects that
 // information into the context that is supplied to the next Spec's `Run`.
 type Result struct {
-	// err is any error that was returned from the Evaluable's execution. This
-	// is guaranteed to be a `gdterrors.RuntimeError`.
-	err error
 	// failures is the collection of error messages from assertion failures
 	// that occurred during Eval(). These are *not* `gdterrors.RuntimeError`.
 	failures []error
@@ -36,17 +26,6 @@ type Result struct {
 	data map[string]interface{}
 }
 
-// HasRuntimeError returns true if the Eval() returned a runtime error, false
-// otherwise.
-func (r *Result) HasRuntimeError() bool {
-	return r.err != nil
-}
-
-// RuntimeError returns the runtime error
-func (r *Result) RuntimeError() error {
-	return r.err
-}
-
 // HasData returns true if any of the run data has been set, false otherwise.
 func (r *Result) HasData() bool {
 	return r.data != nil
@@ -86,19 +65,6 @@ func (r *Result) SetFailures(failures ...error) {
 
 type ResultModifier func(*Result)
 
-// WithRuntimeError modifies the Result with the supplied error
-func WithRuntimeError(err error) ResultModifier {
-	if !errors.Is(err, gdterrors.RuntimeError) {
-		msg := fmt.Sprintf("expected %s to be a gdterrors.RuntimeError", err)
-		// panic here because a plugin author incorrectly implemented their
-		// plugin Spec's Eval() method...
-		panic(msg)
-	}
-	return func(r *Result) {
-		r.err = err
-	}
-}
-
 // WithData modifies the Result with the supplied run data key and value
 func WithData(key string, val interface{}) ResultModifier {
 	return func(r *Result) {