Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions docs/script-format.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ field names the same script would otherwise produce.

### Adding a version

While MAJOR is `0` the format is not stable, and a minor may take a field away
as well as add one -- which is what `0.x` means everywhere else, and what `0.9`
did to `isRegex`. Once `1.0` is cut, removing a field becomes a major bump.

In the same change that alters the schema:

1. Append the new version to `knownVersions`. `CurrentVersion` follows the last
Expand All @@ -75,6 +79,7 @@ but means nothing is worse than no version at all.
| `0.6` | Added `terminal`, and made a pty the default. |
| `0.7` | Added a per-step `timeout`. |
| `0.8` | A runner may use `${runnerDir}` where it names a path. |
| `0.9` | Removed `isRegex`; redactions are the only way to match by pattern. |

### Known limitation

Expand Down Expand Up @@ -386,9 +391,9 @@ way. Normalising differently is how a comparison quietly stops comparing.

### Why not just a regular expression

`isRegex` swaps exact matching for a regular expression over the entire line. To
mask one volatile number with it you have to write a pattern for everything
around that number too:
Until `0.9` a step could set `isRegex` and match the whole line as a regular
expression. To mask one volatile number with it you had to write a pattern for
everything around that number too:

```yaml
- line: '^wrote /[^ ]+ in [0-9]+ms$'
Expand All @@ -398,16 +403,20 @@ around that number too:
which no longer reads as the output it stands for, and no longer says which part
was expected to vary or why. The version with `{{elapsed}}` says both.

`isRegex` still works. It is superseded, not removed: taking it away means every
script using one has to be rewritten, which is a major version bump. Prefer
redactions in anything written from now on.
`isRegex` was removed in `0.9`, and a step's fields are closed, so a script still
carrying one is rejected rather than quietly matched literally:

```
Script validation errors:
runs.0.steps.0: Additional property isRegex is not allowed
```

### Errors

A placeholder naming a redaction the script does not declare is rejected, and
the message lists the ones it does declare. A redaction whose pattern does not
compile is reported once, against the declaration rather than against every step
that refers to it. A step cannot be both `isRegex` and use redactions.
that refers to it.

## Runners

Expand Down
4 changes: 2 additions & 2 deletions internal/play/play_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,15 @@ func TestDoubleInput(t *testing.T) {
assert.Equal(t, 0, exitCode)
}

func TestDoubleInputRegex(t *testing.T) {
func TestDoubleInputRedaction(t *testing.T) {
t.Parallel()

config := createConfig(t, "fixtures/double_input.sh")
regex := regexp.MustCompile(`Sum: \d`)
run := script.Run{Steps: []script.Step{
{Line: "First number: ", Input: "1"},
{Line: "Second number: ", Input: "2"},
{Line: "Sum: \\d", LineRegex: *regex, IsRegex: true},
{Line: "Sum: {{digit}}", LineRegex: *regex, Redacted: true},
}}

exitCode := Run(config.Play, run, config.Logger)
Expand Down
11 changes: 4 additions & 7 deletions internal/play/step_matcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,15 @@ func (matcher *StepMatcher) FailureReport(mode FailureMode, detail string) strin
var report strings.Builder
fmt.Fprintf(&report, "%s: step %d of %d did not match (%s)\n\n",
mode, matcher.currentStepIndex+1, len(matcher.steps), detail)
switch {
case step.Redacted:
if step.Redacted {
fmt.Fprintf(&report, " expected %q (with redactions applied)\n", expected)
case step.IsRegex:
fmt.Fprintf(&report, " expected %q (regular expression)\n", expected)
default:
} else {
fmt.Fprintf(&report, " expected %q\n", expected)
}
fmt.Fprintf(&report, " received %q\n", received)
// A character-level pointer only means something when the two are meant to
// be equal; against a pattern the first differing byte is noise.
if !step.UsesPattern() {
if !step.Redacted {
report.WriteString(differenceMarker(expected, received, len(reportLabelIndent)))
}

Expand Down Expand Up @@ -228,7 +225,7 @@ func (matcher *StepMatcher) Match(char string) error {

func (matcher *StepMatcher) matchLine(step script.Step) (bool, error) {
var matched bool
if step.UsesPattern() {
if step.Redacted {
matched = step.LineRegex.MatchString(matcher.currentLine)
} else {
matched = matcher.currentLine == step.Line
Expand Down
6 changes: 3 additions & 3 deletions internal/play/step_matcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,15 @@ func TestFailureReportOmitsMarkerWhenNothingWasReceived(t *testing.T) {
assert.NotContains(t, report, "^")
}

func TestFailureReportOmitsMarkerForRegexSteps(t *testing.T) {
func TestFailureReportOmitsMarkerForRedactedSteps(t *testing.T) {
t.Parallel()
pattern := regexp.MustCompile(`^Rolled \d+$`)
matcher := newTestMatcher([]script.Step{{Line: pattern.String(), IsRegex: true, LineRegex: *pattern}})
matcher := newTestMatcher([]script.Step{{Line: "Rolled {{n}}", Redacted: true, LineRegex: *pattern}})

feed(t, &matcher, "Rolled twelve")
report := matcher.FailureReport(NoMatch, "nothing matched it within 5s")

assert.Contains(t, report, "(regular expression)")
assert.Contains(t, report, "(with redactions applied)")
// Pointing at the first byte where output differs from a pattern would be
// meaningless.
assert.NotContains(t, report, "first difference")
Expand Down
19 changes: 7 additions & 12 deletions internal/script/redaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@ var redactionPlaceholder = regexp.MustCompile(`\{\{([^{}]*)\}\}`)
// into a regular expression, substituting the named pattern and quoting
// everything around it.
//
// This is what a fixture wants instead of isRegex. Masking one volatile number
// with isRegex means hand-writing a regular expression for the whole line, at
// which point the line stops being readable as the output it stands for and
// the intent behind the mask is lost. A name keeps both: the line still looks
// like what the program prints, and {{seed}} says why that part varies.
// This is the only way a step matches by pattern. The alternative it replaced
// -- declaring the whole line to be a regular expression -- meant hand-writing
// one to mask a single volatile number, at which point the line stopped being
// readable as the output it stands for and the intent behind the mask was
// lost. A name keeps both: the line still looks like what the program prints,
// and {{seed}} says why that part varies.
func compileRedactions(script Script) []string {
var errors []string

Expand All @@ -43,11 +44,6 @@ func compileRedactions(script Script) []string {
}

where := fmt.Sprintf("runs.%d.steps.%d.line", runIndex, stepIndex)
if step.IsRegex {
errors = append(errors, where+": a step cannot both be a regular expression and use redactions")
continue
}

pattern, unknown := buildPattern(step.Line, placeholders, script.Redactions, known)
if len(unknown) > 0 {
errors = append(errors, fmt.Sprintf("%s: %s", where, unknownRedactions(unknown, script.Redactions)))
Expand All @@ -71,8 +67,7 @@ func compileRedactions(script Script) []string {
}

// buildPattern anchors at both ends: a redacted line stands for the whole line
// the program printed, the same as a line matched literally does. isRegex is
// unanchored and stays that way, because scripts already rely on it.
// the program printed, the same as a line matched literally does.
func buildPattern(line string, placeholders [][]int, redactions map[string]string, known map[string]bool) (string, []string) {
var pattern strings.Builder
var unknown []string
Expand Down
32 changes: 1 addition & 31 deletions internal/script/redaction_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ func TestRedactionMatchesTheVaryingPart(t *testing.T) {
step, err := parseWithRedactions(t, `{"rolls": "[0-9]+"}`, `" 7 {{rolls}} "`)
assert.NoError(t, err)
assert.True(t, step.Redacted)
assert.True(t, step.UsesPattern())
assert.True(t, step.LineRegex.MatchString(" 7 807 "))
assert.True(t, step.LineRegex.MatchString(" 7 12 "))
}
Expand Down Expand Up @@ -60,7 +59,7 @@ func TestLineWithoutPlaceholdersStaysLiteral(t *testing.T) {
step, err := parseWithRedactions(t, `{"n": "[0-9]+"}`, `"HOW MANY ROLLS? "`)
assert.NoError(t, err)
assert.False(t, step.Redacted)
assert.False(t, step.UsesPattern())
assert.Empty(t, step.LineRegex.String())
}

func TestUnknownRedactionIsRejected(t *testing.T) {
Expand Down Expand Up @@ -93,19 +92,6 @@ redactions.n: error parsing regexp: missing closing ): ` + "`(unterminated`"
assert.Equal(t, expected, err.Error())
}

func TestStepCannotBeBothRegexAndRedacted(t *testing.T) {
document := `{
"version": "0.4",
"redactions": {"n": "[0-9]+"},
"runs": [{"arguments": [], "exitCode": 0, "steps": [{"line": "seed {{n}}", "isRegex": true}]}]
}`
_, err := ParseScriptFromBytes([]byte(document))
assert.Error(t, err)
expected := `Script validation errors:
runs.0.steps.0.line: a step cannot both be a regular expression and use redactions`
assert.Equal(t, expected, err.Error())
}

// Redactions are declared for the script, so runs being compared normalise the
// same things in the same way rather than each carrying its own copy.
func TestRedactionsApplyToEveryRun(t *testing.T) {
Expand All @@ -124,19 +110,3 @@ func TestRedactionsApplyToEveryRun(t *testing.T) {
assert.True(t, run.Steps[0].LineRegex.MatchString("rolled 6"), run.Name)
}
}

// isRegex still works. It is superseded, not removed: taking it away would
// mean rewriting every script that uses one, which is a major version bump.
func TestIsRegexStillWorks(t *testing.T) {
document := `{
"version": "0.4",
"runs": [{"arguments": [], "exitCode": 0, "steps": [{"line": "hello \\w+", "isRegex": true}]}]
}`
script, err := ParseScriptFromBytes([]byte(document))
assert.NoError(t, err)

step := script.Runs[0].Steps[0]
assert.True(t, step.UsesPattern())
assert.False(t, step.Redacted)
assert.True(t, step.LineRegex.MatchString("hello richard"))
}
3 changes: 1 addition & 2 deletions internal/script/script.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,7 @@ func ParseScriptFromBytes(json []byte) (Script, error) {
return Script{}, err
}

regexErrors = validateRegexes(script.Runs)
regexErrors = append(regexErrors, compileRedactions(script)...)
regexErrors = compileRedactions(script)
regexErrors = append(regexErrors, validateTimeouts(script.Runs)...)
regexErrors = append(regexErrors, ensureNames(script.Runs)...)
if len(regexErrors) == 0 {
Expand Down
23 changes: 11 additions & 12 deletions internal/script/script_parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ func TestBasicScript(t *testing.T) {
"line": "enter your name: ",
"input": "richard"
}, {
"line": "hello \\w+",
"isRegex": true
"line": "hello richard"
}]
}]
}`
Expand All @@ -45,13 +44,11 @@ func TestBasicScript(t *testing.T) {

assert.Equal(t, "enter your name: ", run.Steps[0].Line)
assert.Equal(t, "richard", run.Steps[0].Input)
assert.False(t, run.Steps[0].IsRegex)

stepOne := run.Steps[1]
assert.Equal(t, "hello \\w+", stepOne.Line)
assert.Equal(t, "hello richard", stepOne.Line)
assert.Equal(t, "", stepOne.Input)
assert.True(t, stepOne.IsRegex)
assert.True(t, stepOne.LineRegex.MatchString("hello richard"))
assert.False(t, stepOne.Redacted)
}

func TestGeneratesNames(t *testing.T) {
Expand Down Expand Up @@ -95,8 +92,7 @@ func TestValidationFailsForIncorrectType(t *testing.T) {
"line": "enter your name: ",
"input": "richard"
}, {
"line": "hello \\w+",
"isRegex": true
"line": "hello richard"
}]
}]
}`
Expand Down Expand Up @@ -184,24 +180,27 @@ func TestEveryKnownVersionIsReadable(t *testing.T) {
}
}

func TestHandlesInvalidRegex(t *testing.T) {
// isRegex was removed in 0.9, and a step's properties are closed so that a
// script still carrying one is told about it rather than quietly matched
// literally.
func TestRejectsIsRegex(t *testing.T) {
basicScript := `{
"version": "0.1",
"version": "0.9",
"runs": [{
"arguments": [
"-l"
],
"exitCode": 0,
"steps": [{
"line": "hello (w+",
"line": "hello \\w+",
"isRegex": true
}]
}]
}`
_, err := ParseScriptFromBytes([]byte(basicScript))
assert.Error(t, err)
expected := `Script validation errors:
runs.0.steps.0.line: error parsing regexp: missing closing ): ` + "`hello (w+`"
runs.0.steps.0: Additional property isRegex is not allowed`
assert.Equal(t, expected, err.Error())
}

Expand Down
7 changes: 2 additions & 5 deletions internal/script/script_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,9 @@
"timeout": {
"description": "How long to wait for output while this step is the one being waited for, as a Go duration such as '2m'. Replaces the run's timeout for this step only. Added in 0.7.",
"type": "string"
},
"isRegex": {
"description": "Match the whole line as a regular expression. Superseded by redactions, which keep the line readable as the output it stands for; see docs/script-format.md.",
"type": "boolean"
}
}
},
"additionalProperties": false
}
}
},
Expand Down
7 changes: 0 additions & 7 deletions internal/script/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ type Step struct {
Line string `json:"line"`
LineRegex regexp.Regexp `json:"-"`
Input string `json:"input,omitempty"`
IsRegex bool `json:"isRegex,omitempty"`

// Redacted is set while parsing, for a line that referred to a redaction
// and so matches by pattern rather than by equality. It is not part of the
Expand All @@ -26,12 +25,6 @@ type Step struct {
TimeoutDuration time.Duration `json:"-"`
}

// UsesPattern reports whether the step matches by regular expression rather
// than by exact equality, whichever of the two ways it got there.
func (step Step) UsesPattern() bool {
return step.IsRegex || step.Redacted
}

type Run struct {
Name string `json:"name,omitempty"`
Timestamp time.Time `json:"timestamp"`
Expand Down
21 changes: 0 additions & 21 deletions internal/script/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,34 +3,13 @@ package script
import (
"errors"
"fmt"
"regexp"
"sort"
"strings"
"time"

schema "github.com/xeipuuv/gojsonschema"
)

func validateRegexes(runs []Run) []string {
var regexErrors []string
// validate regexes should they exist
for runIndex, run := range runs {
for stepIndex := range run.Steps {
step := &run.Steps[stepIndex]
if step.IsRegex {
regex, err := regexp.Compile(step.Line)
if err != nil {
regexError := fmt.Sprintf("runs.%d.steps.%d.line: %s", runIndex, stepIndex, err.Error())
regexErrors = append(regexErrors, regexError)
} else {
step.LineRegex = *regex
}
}
}
}
return regexErrors
}

// validateTimeouts parses each step's timeout override. A duration that does
// not parse is reported here rather than at play time, alongside the regexes
// and for the same reason: a script that cannot be run is worth knowing about
Expand Down
6 changes: 5 additions & 1 deletion internal/script/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ import (
// majors are dropped from this list at that point, and the error names the
// last release that could read them.
//
// While MAJOR is 0 the format is not stable and a minor may take something
// away as well as add it, which is what 0.x means everywhere else and what
// 0.9 did to isRegex. Once 1.0 is cut, the rule above is a promise.
//
// Add a version here in the same change that alters the schema, never before.
var knownVersions = []string{"0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8"}
var knownVersions = []string{"0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8", "0.9"}

// CurrentVersion is the version `record` writes and the newest this build
// understands.
Expand Down