Skip to content

Commit 13c1ecd

Browse files
a-hilalya-hilaly
authored
Merge pull request #13 from gdt-dev/on-fail
support on.fail.exec actions
2 parents 789fc84 + d2552cf commit 13c1ecd

File tree

9 files changed

+255
-59
lines changed

9 files changed

+255
-59
lines changed

‎README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,17 @@ the base `Spec` fields listed above):
510510
present in `stderr`.
511511
* `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
512512
least one* must be present in `stderr`.
513+
* `on`: (optional) an object describing actions to take upon certain
514+
conditions.
515+
* `on.fail`: (optional) an object describing an action to take when any
516+
assertion fails for the test action.
517+
* `on.fail.exec`: a string with the exact command to execute upon test
518+
assertion failure. You may execute more than one command but must include the
519+
`on.fail.shell` field to indicate that the command should be run in a shell.
520+
* `on.fail.shell`: (optional) a string with the specific shell to use in executing the
521+
command to run upon test assertion failure. If empty (the default), no shell
522+
is used to execute the command and instead the operating system's `exec` family
523+
of calls is used.
513524

514525
[execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
515526
[pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26

‎plugin/exec/action.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Use and distribution licensed under the Apache license version 2.
2+
//
3+
// See the COPYING file in the root project directory for full text.
4+
5+
package exec
6+
7+
import (
8+
"bytes"
9+
"context"
10+
"os/exec"
11+
"testing"
12+
13+
gdtcontext "github.com/gdt-dev/gdt/context"
14+
"github.com/gdt-dev/gdt/debug"
15+
gdterrors "github.com/gdt-dev/gdt/errors"
16+
"github.com/google/shlex"
17+
)
18+
19+
// Action describes a single execution of one or more commands via the
20+
// operating system's `exec` family of functions.
21+
type Action struct {
22+
// Exec is the exact command to execute.
23+
//
24+
// You may execute more than one command but must include the `shell` field
25+
// to indicate that the command should be run in a shell. It is best
26+
// practice, however, to simply use multiple `exec` specs instead of
27+
// executing multiple commands in a single shell call.
28+
Exec string `yaml:"exec"`
29+
// Shell is the specific shell to use in executing the command. If empty
30+
// (the default), no shell is used to execute the command and instead the
31+
// operating system's `exec` family of calls is used.
32+
Shell string `yaml:"shell,omitempty"`
33+
}
34+
35+
// Do performs a single command or shell execution returning the corresponding
36+
// exit code and any runtime error. The `outbuf` and `errbuf` buffers will be
37+
// filled with the contents of the command's stdout and stderr pipes
38+
// respectively.
39+
func (a *Action) Do(
40+
ctx context.Context,
41+
t *testing.T,
42+
outbuf *bytes.Buffer,
43+
errbuf *bytes.Buffer,
44+
exitcode *int,
45+
) error {
46+
var target string
47+
var args []string
48+
if a.Shell == "" {
49+
// Parse time already validated exec string parses into valid shell
50+
// args
51+
args, _ = shlex.Split(a.Exec)
52+
target = args[0]
53+
args = args[1:]
54+
} else {
55+
target = a.Shell
56+
args = []string{"-c", a.Exec}
57+
}
58+
59+
debug.Println(ctx, t, "exec: %s %s", target, args)
60+
61+
var cmd *exec.Cmd
62+
cmd = exec.CommandContext(ctx, target, args...)
63+
64+
outpipe, err := cmd.StdoutPipe()
65+
if err != nil {
66+
return err
67+
}
68+
errpipe, err := cmd.StderrPipe()
69+
if err != nil {
70+
return err
71+
}
72+
73+
err = cmd.Start()
74+
if gdtcontext.TimedOut(ctx, err) {
75+
return gdterrors.ErrTimeoutExceeded
76+
}
77+
if err != nil {
78+
return err
79+
}
80+
if outbuf != nil {
81+
outbuf.ReadFrom(outpipe)
82+
}
83+
if errbuf != nil {
84+
errbuf.ReadFrom(errpipe)
85+
}
86+
87+
err = cmd.Wait()
88+
if gdtcontext.TimedOut(ctx, err) {
89+
return gdterrors.ErrTimeoutExceeded
90+
}
91+
if err != nil && exitcode != nil {
92+
eerr, _ := err.(*exec.ExitError)
93+
ec := eerr.ExitCode()
94+
*exitcode = ec
95+
}
96+
return nil
97+
}

‎plugin/exec/eval.go‎

Lines changed: 25 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,8 @@ package exec
77
import (
88
"bytes"
99
"context"
10-
"os/exec"
1110
"testing"
1211

13-
"github.com/google/shlex"
14-
15-
gdtcontext "github.com/gdt-dev/gdt/context"
1612
"github.com/gdt-dev/gdt/debug"
1713
gdterrors "github.com/gdt-dev/gdt/errors"
1814
"github.com/gdt-dev/gdt/result"
@@ -25,57 +21,39 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
2521
outbuf := &bytes.Buffer{}
2622
errbuf := &bytes.Buffer{}
2723

28-
var err error
29-
var cmd *exec.Cmd
30-
var target string
31-
var args []string
32-
if s.Shell == "" {
33-
// Parse time already validated exec string parses into valid shell
34-
// args
35-
args, _ = shlex.Split(s.Exec)
36-
target = args[0]
37-
args = args[1:]
38-
} else {
39-
target = s.Shell
40-
args = []string{"-c", s.Exec}
41-
}
42-
43-
debug.Println(ctx, t, "exec: %s %s", target, args)
44-
cmd = exec.CommandContext(ctx, target, args...)
45-
46-
outpipe, err := cmd.StdoutPipe()
47-
if err != nil {
48-
return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
49-
}
50-
errpipe, err := cmd.StderrPipe()
51-
if err != nil {
52-
return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
53-
}
24+
var ec int
5425

55-
err = cmd.Start()
56-
if gdtcontext.TimedOut(ctx, err) {
57-
return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
58-
}
59-
if err != nil {
26+
if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
27+
if err == gdterrors.ErrTimeoutExceeded {
28+
return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
29+
}
6030
return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
6131
}
62-
outbuf.ReadFrom(outpipe)
63-
errbuf.ReadFrom(errpipe)
64-
65-
err = cmd.Wait()
66-
if gdtcontext.TimedOut(ctx, err) {
67-
return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
68-
}
69-
ec := 0
70-
if err != nil {
71-
eerr, _ := err.(*exec.ExitError)
72-
ec = eerr.ExitCode()
73-
}
7432
a := newAssertions(s.Assert, ec, outbuf, errbuf)
7533
if !a.OK() {
7634
for _, fail := range a.Failures() {
7735
t.Error(fail)
7836
}
37+
if s.On != nil {
38+
if s.On.Fail != nil {
39+
outbuf.Reset()
40+
errbuf.Reset()
41+
err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
42+
if err != nil {
43+
debug.Println(ctx, t, "error in on.fail.exec: %s", err)
44+
}
45+
if outbuf.Len() > 0 {
46+
debug.Println(
47+
ctx, t, "on.fail.exec: stdout: %s", outbuf.String(),
48+
)
49+
}
50+
if errbuf.Len() > 0 {
51+
debug.Println(
52+
ctx, t, "on.fail.exec: stderr: %s", errbuf.String(),
53+
)
54+
}
55+
}
56+
}
7957
}
8058
return result.New(result.WithFailures(a.Failures()...))
8159
}

‎plugin/exec/eval_test.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,3 +235,50 @@ func TestTimeoutCascade(t *testing.T) {
235235
require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
236236
require.Contains(debugout, "using timeout of 20ms (expected: true)")
237237
}
238+
239+
// Unfortunately there's not really any good way of testing things like this
240+
// except by manually causing an assertion to fail in the test case and
241+
// checking to see if the `on.fail` action was taken and debug output emitted
242+
// to the console.
243+
//
244+
// When I change the `testdata/on-fail-exec.yaml` file to have a failed
245+
// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the
246+
// correct behaviour:
247+
//
248+
// === RUN TestOnFail
249+
// === RUN TestOnFail/on-fail-exec
250+
//
251+
// action.go:59: exec: echo [cat]
252+
// eval.go:35: assertion failed: not equal: expected dat but got cat
253+
// action.go:59: exec: echo [bad kitty]
254+
// eval.go:46: on.fail.exec: stdout: bad kitty
255+
//
256+
// === NAME TestOnFail
257+
//
258+
// eval_test.go:256:
259+
// Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
260+
// Error: Should be false
261+
// Test: TestOnFail
262+
//
263+
// --- FAIL: TestOnFail (0.00s)
264+
//
265+
// --- FAIL: TestOnFail/on-fail-exec (0.00s)
266+
func TestOnFail(t *testing.T) {
267+
require := require.New(t)
268+
269+
fp := filepath.Join("testdata", "on-fail-exec.yaml")
270+
f, err := os.Open(fp)
271+
require.Nil(err)
272+
273+
s, err := scenario.FromReader(
274+
f,
275+
scenario.WithPath(fp),
276+
)
277+
require.Nil(err)
278+
require.NotNil(s)
279+
280+
ctx := gdtcontext.New(gdtcontext.WithDebug())
281+
err = s.Run(ctx, t)
282+
require.Nil(err)
283+
require.False(t.Failed())
284+
}

‎plugin/exec/on.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Use and distribution licensed under the Apache license version 2.
2+
//
3+
// See the COPYING file in the root project directory for full text.
4+
5+
package exec
6+
7+
// On describes actions that can be taken upon certain conditions.
8+
type On struct {
9+
// Fail contains one or more actions to take if any of a Spec's assertions
10+
// fail.
11+
//
12+
// For example, if you wanted to grep a log file in the event that no
13+
// connectivity on a particular IP:PORT combination could be made you might
14+
// do this:
15+
//
16+
// ```yaml
17+
// tests:
18+
// - exec: nc -z $HOST $PORT
19+
// on:
20+
// fail:
21+
// exec: grep ERROR /var/log/myapp.log
22+
// ```
23+
//
24+
// The `grep ERROR /var/log/myapp.log` command will only be executed if
25+
// there is no connectivity to $HOST:$PORT and the results of that grep
26+
// will be directed to the test's output. You can use the `gdt.WithDebug()`
27+
// function to configure additional `io.Writer`s to direct this output to.
28+
Fail *Action `yaml:"fail,omitempty"`
29+
}

‎plugin/exec/parse.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,15 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
7272
return err
7373
}
7474
s.Assert = e
75+
case "on":
76+
if valNode.Kind != yaml.MappingNode {
77+
return errors.ExpectedMapAt(valNode)
78+
}
79+
var o *On
80+
if err := valNode.Decode(&o); err != nil {
81+
return err
82+
}
83+
s.On = o
7584
default:
7685
if lo.Contains(gdttypes.BaseSpecFields, key) {
7786
continue

‎plugin/exec/parse_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ func TestSimpleCommand(t *testing.T) {
5757
Index: 0,
5858
Defaults: &gdttypes.Defaults{},
5959
},
60-
Exec: "ls",
60+
Action: gdtexec.Action{
61+
Exec: "ls",
62+
},
6163
},
6264
}
6365
assert.Equal(expTests, s.Tests)

‎plugin/exec/spec.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,11 @@ import (
1212
// operating system's `exec` family of functions.
1313
type Spec struct {
1414
gdttypes.Spec
15-
// Exec is the exact command to execute.
16-
//
17-
// You may execute more than one command but must include the `shell` field
18-
// to indicate that the command should be run in a shell. It is best
19-
// practice, however, to simply use multiple `exec` specs instead of
20-
// executing multiple commands in a single shell call.
21-
Exec string `yaml:"exec"`
22-
// Shell is the specific shell to use in executing the command. If empty
23-
// (the default), no shell is used to execute the command and instead the
24-
// operating system's `exec` family of calls is used.
25-
Shell string `yaml:"shell,omitempty"`
15+
Action
2616
// Assert is an object containing the conditions that the Spec will assert.
2717
Assert *Expect `yaml:"assert,omitempty"`
18+
// On is an object containing actions to take upon certain conditions.
19+
On *On `yaml:"on,omitempty"`
2820
}
2921

3022
func (s *Spec) SetBase(b gdttypes.Spec) {

‎plugin/exec/testdata/on-fail-exec.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: on-fail-exec
2+
description: a scenario that has an on.fail.exec clause
3+
tests:
4+
- exec: echo "cat"
5+
assert:
6+
out:
7+
is: cat
8+
# Unfortunately there's not really any good way of testing things like this
9+
# except by manually causing an assertion to fail in the test case and checking
10+
# to see if the `on.fail` action was taken and debug output emitted to the
11+
# console.
12+
#
13+
# When I change `assert.out.is` above to "dat" instead of "cat", I get the
14+
# correct behaviour:
15+
#
16+
# === RUN TestOnFail
17+
# === RUN TestOnFail/on-fail-exec
18+
# action.go:59: exec: echo [cat]
19+
# eval.go:35: assertion failed: not equal: expected dat but got cat
20+
# action.go:59: exec: echo [bad kitty]
21+
# eval.go:46: on.fail.exec: stdout: bad kitty
22+
# === NAME TestOnFail
23+
# eval_test.go:256:
24+
# Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
25+
# Error: Should be false
26+
# Test: TestOnFail
27+
# --- FAIL: TestOnFail (0.00s)
28+
# --- FAIL: TestOnFail/on-fail-exec (0.00s)
29+
on:
30+
fail:
31+
exec: echo "bad kitty"

0 commit comments

Comments
 (0)