From fef7ac067de4821affe98701e50ad9145186c61e Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 22:29:48 +0100 Subject: [PATCH] compare implementations aligned by step A script with more than one run is asking where the implementations stop agreeing, and until now it answered with a stack of separate failure reports and a tally of how many failed. Doing the comparison was left to whoever read them. It now ends with the comparison itself: every port lined up against the first run at the step where it left it, with both sides quoted under one column so the difference is read down the page. The divergence becomes a value rather than a string built where the failure happened, which is what makes aligning several runs possible at all -- a step has to be a number something else can group by. Ports that left at the same step, the same way, printing the same thing are one finding: a dozen ports of the same BASIC program missing a leading space is one bug repeated twelve times, and a report that says it twelve times is one nobody reads to the end. Three things are deliberately not divergences. A port that plays a different transcript is not compared at all, because step 7 of one transcript has nothing to do with step 7 of another. A port that never ran has not disagreed with anything. And when the reference no longer matches its own transcript, nothing is compared: the environment has moved, and every port's divergence would be measured against something that is itself wrong. A run that could not be started now says so on stderr rather than only through a logger that is off by default -- the comparison names it as one that could not be played, which invites the question. Closes #16 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- README.md | 24 +++ docs/comparison.md | 126 ++++++++++++ docs/script-format.md | 8 +- internal/play/comparison.go | 243 +++++++++++++++++++++++ internal/play/comparison_test.go | 242 ++++++++++++++++++++++ internal/play/divergence.go | 171 ++++++++++++++++ internal/play/play.go | 68 +++++-- internal/play/play_integration_test.go | 62 +++--- internal/play/runner_integration_test.go | 14 +- internal/play/runs.go | 25 ++- internal/play/step_matcher.go | 120 +++-------- 11 files changed, 932 insertions(+), 171 deletions(-) create mode 100644 docs/comparison.md create mode 100644 internal/play/comparison.go create mode 100644 internal/play/comparison_test.go create mode 100644 internal/play/divergence.go diff --git a/README.md b/README.md index 7867289..3e1c2dc 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,30 @@ see [docs/script-format.md](docs/script-format.md#failure-modes). A run that times out kills the program before writing its report, and under a pty takes anything the program started with it. +### Comparing implementations + +A script with more than one run is asking where the implementations stop +agreeing, so it ends with a comparison rather than a stack of separate failure +reports. The first run is the reference; every other one is lined up against it +at the step where it left it, and ports that got the same thing wrong are one +finding rather than several: + +``` +=== comparison against basic === + +step 1 of 2 (exited-early) + basic "YOU HAVE 3 GUESSES LEFT" + ruby, python "You have 3 guesses left" + "TRY AGAIN" + +matched: perl +could not be played: go +``` + +A port that plays a different transcript is not compared, and neither is one +that never ran — neither of them has disagreed with anything. See +[docs/comparison.md](docs/comparison.md). + ### Development ``` diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 0000000..14e1ad1 --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,126 @@ +# Comparing implementations + +A script with one run answers a question about one program. A script with more +than one run is asking a different question — *where do these implementations +stop agreeing?* — and the answer to that is not a verdict on each of them +separately. Playing ten ports and printing ten failure reports leaves the +reader to do the comparison by hand, which is the work the script was written +to do for them. + +So a multi-run script ends with a comparison: every port lined up against the +reference at the step where it left it. + +``` +=== comparison against basic === + +step 1 of 2 (exited-early) + basic "YOU HAVE 3 GUESSES LEFT" + ruby, python "You have 3 guesses left" + "TRY AGAIN" + +step 2 of 2 (exited-early) + basic "TRY AGAIN" + node "" + +matched: perl +``` + +The comparison is written to stderr after the last run, and does not replace +the individual failure reports above it — those say *what* each run did, in +full, and this says how the runs relate to each other. + +## The first run is the reference + +Not a rule imposed on the format so much as a description of how such a script +comes about: it is recorded from the implementation whose behaviour is being +treated as correct, and the ports are added underneath. + +The reference is played like every other run, which matters. If it no longer +matches its own transcript then the environment has moved — a different +interpreter build, a different locale, a tape that no longer fits — and every +port's "divergence" is being measured against something that is itself wrong. +When that happens nothing is compared: + +``` +=== comparison against basic === + +basic did not match its own transcript, so there was nothing for the +ports to be compared against. Its own failure is above. +``` + +## Ports that got the same thing wrong are one finding + +Ports are grouped by where they diverged, how, and what they printed instead. +They are not independent observations: a dozen ports of the same BASIC program +missing the leading space BASIC prints before a positive number is one bug +repeated twelve times, and a report that says it twelve times is a report +nobody reads to the end. + +Findings are ordered by step, earliest first. A later divergence is often a +consequence of an earlier one — the port that got the prompt wrong is the port +that then answered the wrong question — so the first one is the one to read, +and it should not be somewhere in the middle because of which port happened to +hit it. + +Every quoted line uses the same column across the whole report, so the +differences are read down the page rather than re-found at each step. + +## What is shown against the reference + +The reference's expected line, and then everything the ports printed since they +last agreed with the transcript — not one chosen line. + +Which of several lines was *meant* to be the step is exactly the question a +divergence raises, so the report does not answer it. When a port printed one +line, there is nothing to choose between and the caret points at the byte where +it went wrong: + +``` +step 1 of 2 (no-match) + basic "HOW MANY ROLLS? " + node "HOW MANY ROLLS?" + ^ received output ends here +``` + +When it printed several, they are all shown and none is pointed at. + +A step matched through [redactions](script-format.md#redactions) shows the +expected line as written, marked `(with redactions applied)`, and gets no +caret: the difference is against a pattern, and a column number would name a +position in the pattern rather than in the output. + +A run that matched every step and then went wrong — `hung`, or +`wrong-exit-code` — has no line left to put opposite the reference, so it says +what it did instead of finishing: + +``` +after all 12 steps (hung) + ruby the executable had not exited after 30s +``` + +## Ports that were not compared + +Three things are not divergences, and saying they were would be the kind of +confident wrong answer this tool exists to catch: + +| Line | What it means | +| --- | --- | +| `matched: ...` | played the reference's transcript and agreed with all of it | +| `not comparable, they play a different transcript: ...` | its steps are not the reference's steps | +| `could not be played: ...` | it never ran — a missing executable, an unreadable tape | + +A run is comparable when it expects the same lines and sends the same input. +Everything else about it — which executable, which arguments, which +environment, which terminal — is what the comparison is *about*, and two runs +identical in those respects would have nothing to say to each other. + +Step 7 of one transcript has nothing to do with step 7 of another, so lining up +two different transcripts would report a difference that is an artefact of the +script rather than of the implementations. Two runs that legitimately expect +different output belong in two scripts. + +## Exit codes are unchanged + +`prescript` still exits with the first non-zero code any run produced. The +comparison is a report, not a verdict: it says where the implementations parted +company, and what to do about that is the reader's call. diff --git a/docs/script-format.md b/docs/script-format.md index c79610e..22bca17 100644 --- a/docs/script-format.md +++ b/docs/script-format.md @@ -230,8 +230,12 @@ executable, and they share the steps they are expected to produce. Every run is played, even after one has failed. Stopping at the first failure would withhold the comparison the script was written to make: knowing the first of three ports is wrong says nothing about the other two. `prescript` exits with -the first non-zero code and, when there was more than one run, ends with a line -naming which of them failed. +the first non-zero code. + +The first run is the reference, and a script with more than one ends with a +comparison: every port lined up against that reference at the step where it +left it, with ports that got the same thing wrong grouped into one finding. See +[comparison.md](comparison.md). ### Names diff --git a/internal/play/comparison.go b/internal/play/comparison.go new file mode 100644 index 0000000..11b8c8b --- /dev/null +++ b/internal/play/comparison.go @@ -0,0 +1,243 @@ +package play + +import ( + "fmt" + "sort" + "strings" + + "github.com/rjnienaber/prescript/internal/script" + "github.com/rjnienaber/prescript/internal/utils" +) + +// Comparison is what a script with more than one run was written to produce: +// not a verdict on each implementation separately, but the places where they +// stopped agreeing. +// +// The first run is the reference. That is not a rule imposed on the format so +// much as a description of how such a script comes about -- it is recorded +// from the implementation whose behaviour is being treated as correct, and the +// ports are added underneath. It also means the reference is played like any +// other run, which matters: if it no longer matches its own transcript then +// the environment has moved, and every port's "divergence" is measured against +// something that is itself wrong. +type Comparison struct { + Reference string + + // ReferenceFailure is nil when the reference matched its transcript. When + // it is set, nothing else here was compared. + ReferenceFailure *Divergence + + Groups []DivergenceGroup + + // Matched, Incomparable and NotPlayed are port names. A port is + // incomparable when its steps are not the reference's steps: step 7 of one + // transcript has nothing to do with step 7 of another, and lining the two + // up would produce a difference that is an artefact of the script. + Matched []string + Incomparable []string + NotPlayed []string +} + +// DivergenceGroup is one finding: the ports that left the reference at the +// same step, in the same way, printing the same thing. +// +// Ports are grouped rather than listed one by one because they are not +// independent observations. A dozen ports of the same program getting BASIC's +// leading space before a number wrong is one bug repeated twelve times, and a +// report that says it twelve times is a report nobody reads to the end. +type DivergenceGroup struct { + StepIndex int + StepCount int + Mode FailureMode + Expected string + Redacted bool + + // Received is everything the ports printed since they last agreed with the + // transcript. When every step matched and the run went wrong afterwards + // there is nothing left to have printed, and this holds the detail -- what + // the program did instead of finishing -- so the group still says + // something. + Received []string + Matched bool + + Ports []string +} + +// Compare lines up what each run did against the reference's transcript. It +// reads outcomes rather than replaying anything: the runs have already been +// played, in the order the script names them, and outcomes[i] belongs to +// runs[i]. +func Compare(runs []script.Run, outcomes []Outcome) Comparison { + comparison := Comparison{Reference: outcomes[0].Name, ReferenceFailure: outcomes[0].Failure} + if comparison.ReferenceFailure != nil { + return comparison + } + + reference := transcript(runs[0]) + // Keyed by what makes two divergences the same finding. + index := map[string]int{} + + for position, outcome := range outcomes[1:] { + run := runs[position+1] + + switch { + case transcript(run) != reference: + comparison.Incomparable = append(comparison.Incomparable, outcome.Name) + case outcome.Failure == nil && outcome.ExitCode == utils.SUCCESS: + comparison.Matched = append(comparison.Matched, outcome.Name) + case outcome.Failure == nil: + comparison.NotPlayed = append(comparison.NotPlayed, outcome.Name) + default: + comparison.add(index, outcome.Name, *outcome.Failure) + } + } + + // Earliest step first, and within a step the order the ports appear in the + // script. A later divergence is often a consequence of an earlier one -- + // the port that got the prompt wrong is the port that then answered the + // wrong question -- so the first one is the one to read, and it should not + // be somewhere in the middle because of which port happened to hit it. + sort.SliceStable(comparison.Groups, func(i, j int) bool { + return comparison.Groups[i].StepIndex < comparison.Groups[j].StepIndex + }) + + return comparison +} + +func (comparison *Comparison) add(index map[string]int, port string, divergence Divergence) { + group := DivergenceGroup{ + StepIndex: divergence.StepIndex, + StepCount: divergence.StepCount, + Mode: divergence.Mode, + Expected: divergence.Expected, + Redacted: divergence.Redacted, + Received: divergence.Lines(), + Matched: divergence.Matched(), + } + if group.Matched { + group.Received = []string{divergence.Detail} + } + + key := fmt.Sprintf("%d\x00%s\x00%s", group.StepIndex, group.Mode, + strings.Join(group.Received, "\x00")) + if at, seen := index[key]; seen { + comparison.Groups[at].Ports = append(comparison.Groups[at].Ports, port) + return + } + + group.Ports = []string{port} + index[key] = len(comparison.Groups) + comparison.Groups = append(comparison.Groups, group) +} + +// transcript is what makes two runs comparable: the same expectations, fed the +// same input. Everything else about a run -- which executable, which +// environment, which terminal -- is exactly what the comparison is about, and +// two runs identical in those respects would have nothing to say. +func transcript(run script.Run) string { + var signature strings.Builder + for _, step := range run.Steps { + signature.WriteString(step.Line) + signature.WriteString("\x00") + signature.WriteString(step.Input) + signature.WriteString("\x00\x00") + } + return signature.String() +} + +// Report writes the comparison out for a person to read. +func (comparison Comparison) Report() string { + var report strings.Builder + fmt.Fprintf(&report, "\n=== comparison against %s ===\n\n", comparison.Reference) + + if comparison.ReferenceFailure != nil { + fmt.Fprintf(&report, "%s did not match its own transcript, so there was nothing for the\n"+ + "ports to be compared against. Its own failure is above.\n", comparison.Reference) + return report.String() + } + + // One column width for the whole report rather than one per group, so the + // quoted lines stay under each other and a reader compares down the page + // instead of re-finding the column at every step. + width := comparison.labelWidth() + for _, group := range comparison.Groups { + report.WriteString(group.report(comparison.Reference, width)) + report.WriteString("\n") + } + + // Said only of the ports that were actually compared: one that plays a + // different transcript, or that never started, has not agreed with the + // reference -- it has not been asked. Whichever line is used has to be + // true on its own, because it is the line that gets quoted. + if len(comparison.Groups) == 0 && len(comparison.Matched) > 0 { + if len(comparison.Incomparable) > 0 || len(comparison.NotPlayed) > 0 { + report.WriteString("none of the ports that were compared diverged from the reference\n") + } else { + report.WriteString("no port diverged from the reference\n") + } + } + + writeNames(&report, "matched", comparison.Matched) + writeNames(&report, "not comparable, they play a different transcript", comparison.Incomparable) + writeNames(&report, "could not be played", comparison.NotPlayed) + + return report.String() +} + +func (comparison Comparison) labelWidth() int { + width := len(comparison.Reference) + for _, group := range comparison.Groups { + if label := len(strings.Join(group.Ports, ", ")); label > width { + width = label + } + } + return width +} + +func writeNames(report *strings.Builder, label string, names []string) { + if len(names) == 0 { + return + } + fmt.Fprintf(report, "%s: %s\n", label, strings.Join(names, ", ")) +} + +// report lays a group out with the reference's line above the ports' output, +// so the difference is read down the page rather than across it. Both are +// quoted for the same reason a single failure report quotes them: a trailing +// space is the difference that matters most and shows least. +func (group DivergenceGroup) report(reference string, width int) string { + ports := strings.Join(group.Ports, ", ") + indent := len(" ") + width + len(" ") + + var report strings.Builder + if group.Matched { + fmt.Fprintf(&report, "after all %d steps (%s)\n", group.StepCount, group.Mode) + fmt.Fprintf(&report, " %-*s %s\n", width, ports, strings.Join(group.Received, " ")) + return report.String() + } + + fmt.Fprintf(&report, "step %d of %d (%s)\n", group.StepIndex+1, group.StepCount, group.Mode) + if group.Redacted { + fmt.Fprintf(&report, " %-*s %q (with redactions applied)\n", width, reference, group.Expected) + } else { + fmt.Fprintf(&report, " %-*s %q\n", width, reference, group.Expected) + } + + for index, line := range group.Received { + label := "" + if index == 0 { + label = ports + } + fmt.Fprintf(&report, " %-*s %q\n", width, label, line) + } + + // A caret under one of several lines would be asserting which of them was + // meant to be the step, and that is the thing in question. With one line + // there is nothing to assert, and pointing at the byte where it went wrong + // is the whole reason a report quotes it. + if !group.Redacted && len(group.Received) == 1 { + report.WriteString(differenceMarker(group.Expected, group.Received[0], indent)) + } + + return report.String() +} diff --git a/internal/play/comparison_test.go b/internal/play/comparison_test.go new file mode 100644 index 0000000..b45be0e --- /dev/null +++ b/internal/play/comparison_test.go @@ -0,0 +1,242 @@ +package play + +import ( + "strings" + "testing" + + "github.com/rjnienaber/prescript/internal/script" + "github.com/rjnienaber/prescript/internal/utils" + "github.com/stretchr/testify/assert" +) + +// A transcript two runs can share, so a comparison between them is about the +// implementations rather than about the scripts. +func transcriptSteps() []script.Step { + return []script.Step{ + {Line: "HOW MANY ROLLS? ", Input: "5000"}, + {Line: " 7 807 "}, + } +} + +func comparableRun(name string) script.Run { + return script.Run{Name: name, Steps: transcriptSteps()} +} + +func matchedOutcome(name string) Outcome { + return Outcome{Name: name, ExitCode: utils.SUCCESS} +} + +func divergedOutcome(name string, divergence Divergence) Outcome { + return Outcome{Name: name, ExitCode: utils.CLI_ERROR, Failure: &divergence} +} + +func atStep(index int, received ...string) Divergence { + divergence := Divergence{ + Mode: NoMatch, + Detail: "nothing matched it within 5s", + StepIndex: index, + StepCount: len(transcriptSteps()), + Expected: transcriptSteps()[index].Line, + Received: received[len(received)-1], + } + if len(received) > 1 { + divergence.Context = received[:len(received)-1] + } + return divergence +} + +// The point of grouping: ports of the same program get the same thing wrong, +// and a report that says it once is a finding while a report that says it +// three times is a list. +func TestPortsThatDivergeIdenticallyAreOneFinding(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("python")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + divergedOutcome("python", atStep(1, "7 807")), + } + + comparison := Compare(runs, outcomes) + + assert.Len(t, comparison.Groups, 1) + assert.Equal(t, []string{"ruby", "python"}, comparison.Groups[0].Ports) + assert.Contains(t, comparison.Report(), ` ruby, python "7 807"`) +} + +func TestPortsThatDivergeDifferentlyAreSeparateFindings(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("python")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + divergedOutcome("python", atStep(1, "7 807")), + } + + comparison := Compare(runs, outcomes) + + assert.Len(t, comparison.Groups, 2) +} + +// The earliest divergence is the one to read: a later one is often what the +// earlier one caused. +func TestFindingsAreOrderedByStep(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("node")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + divergedOutcome("node", atStep(0, "HOW MANY ROLLS?")), + } + + comparison := Compare(runs, outcomes) + + assert.Equal(t, []int{0, 1}, []int{comparison.Groups[0].StepIndex, comparison.Groups[1].StepIndex}) + report := comparison.Report() + assert.Less(t, strings.Index(report, "step 1 of 2"), strings.Index(report, "step 2 of 2")) +} + +// Both sides, aligned, with the reference above: what the issue asked for. +func TestReportShowsBothSidesAgainstTheReference(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{matchedOutcome("basic"), divergedOutcome("ruby", atStep(1, "7 807"))} + + report := Compare(runs, outcomes).Report() + + assert.Contains(t, report, "=== comparison against basic ===") + assert.Contains(t, report, "step 2 of 2 (no-match)") + assert.Contains(t, report, ` basic " 7 807 "`) + assert.Contains(t, report, ` ruby "7 807"`) + assert.Contains(t, report, "^ first difference") +} + +// A port that printed several lines where one was expected: which of them was +// meant to be the step is the question, so all of them are shown and none is +// pointed at. +func TestSeveralReceivedLinesAreShownWithoutAMarker(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807", "and then some")), + } + + report := Compare(runs, outcomes).Report() + + assert.Contains(t, report, ` ruby "7 807"`) + assert.Contains(t, report, ` "and then some"`) + assert.NotContains(t, report, "first difference") +} + +// Step 7 of one transcript has nothing to do with step 7 of another, so lining +// the two up would report a difference that is an artefact of the script. +func TestAPortPlayingADifferentTranscriptIsNotCompared(t *testing.T) { + t.Parallel() + + other := script.Run{Name: "lua", Steps: []script.Step{{Line: "something else"}}} + runs := []script.Run{comparableRun("basic"), other} + outcomes := []Outcome{matchedOutcome("basic"), divergedOutcome("lua", atStep(0, "nope"))} + + comparison := Compare(runs, outcomes) + + assert.Empty(t, comparison.Groups) + assert.Equal(t, []string{"lua"}, comparison.Incomparable) + assert.Contains(t, comparison.Report(), "not comparable, they play a different transcript: lua") +} + +// A port that never started has not disagreed with anything, and saying it did +// would be the kind of confident wrong answer this tool exists to catch. +func TestAPortThatNeverRanIsNotADivergence(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("go")} + outcomes := []Outcome{matchedOutcome("basic"), {Name: "go", ExitCode: utils.USER_ERROR}} + + comparison := Compare(runs, outcomes) + + assert.Empty(t, comparison.Groups) + assert.Equal(t, []string{"go"}, comparison.NotPlayed) + assert.Contains(t, comparison.Report(), "could not be played: go") +} + +// If the reference no longer matches its own transcript then the environment +// has moved, and every port's divergence is measured against something that is +// itself wrong. +func TestNothingIsComparedWhenTheReferenceItselfDiverged(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + divergedOutcome("basic", atStep(0, "HOW MANY ROLLS?")), + divergedOutcome("ruby", atStep(1, "7 807")), + } + + comparison := Compare(runs, outcomes) + + assert.NotNil(t, comparison.ReferenceFailure) + assert.Empty(t, comparison.Groups) + report := comparison.Report() + assert.Contains(t, report, "basic did not match its own transcript") + assert.NotContains(t, report, "ruby") +} + +func TestEveryPortMatching(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("python")} + outcomes := []Outcome{matchedOutcome("basic"), matchedOutcome("ruby"), matchedOutcome("python")} + + report := Compare(runs, outcomes).Report() + + assert.Contains(t, report, "no port diverged from the reference") + assert.Contains(t, report, "matched: ruby, python") +} + +// A run that got through every step and then hung has no line to put opposite +// the reference, so it says what it did instead of finishing. +func TestARunThatWentWrongAfterTheLastStep(t *testing.T) { + t.Parallel() + + hung := Divergence{ + Mode: Hung, + Detail: "the executable had not exited after 5s", + StepIndex: 2, + StepCount: 2, + } + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{matchedOutcome("basic"), divergedOutcome("ruby", hung)} + + report := Compare(runs, outcomes).Report() + + assert.Contains(t, report, "after all 2 steps (hung)") + assert.Contains(t, report, "the executable had not exited after 5s") +} + +// Columns are shared across findings so the quoted lines stay under each +// other, and a reader compares down the page instead of re-finding the column. +func TestColumnsLineUpAcrossFindings(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("a-long-port-name")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + divergedOutcome("a-long-port-name", atStep(0, "HOW MANY ROLLS?")), + } + + report := Compare(runs, outcomes).Report() + + columns := map[int]bool{} + for _, line := range strings.Split(report, "\n") { + if quote := strings.Index(line, `"`); strings.HasPrefix(line, " ") && quote > 0 { + columns[quote] = true + } + } + assert.Len(t, columns, 1) +} diff --git a/internal/play/divergence.go b/internal/play/divergence.go new file mode 100644 index 0000000..e5405f3 --- /dev/null +++ b/internal/play/divergence.go @@ -0,0 +1,171 @@ +package play + +import ( + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +// Divergence is the point at which a run stopped agreeing with the transcript +// it was played against: which step, what that step wanted, and what arrived +// instead. +// +// It exists as a value rather than as the string the failure report used to +// build directly, because one run's failure is a report and several runs' +// failures are a comparison. Aligning three ports at the step where each of +// them left the reference is only possible if that step is a number something +// else can group by, and the same is true of counting a corpus by what went +// wrong. The report is then a rendering of this, so the two cannot drift. +type Divergence struct { + Mode FailureMode + Detail string + + // StepIndex is zero-based, and equals StepCount when every step matched + // and the run went wrong afterwards -- it hung, or exited with the wrong + // code. Matched says which of the two this is. + StepIndex int + StepCount int + + // Expected and Received are empty when every step matched: there is no + // step left to have wanted anything. + Expected string + Received string + + // Redacted repeats the step's own flag, because it decides both how the + // expected line is labelled and whether pointing at a first differing + // byte would mean anything. + Redacted bool + + // Context is the output since the last matched step, excluding the line + // already shown as Received. + Context []string +} + +// Matched is true when the run got through every step, and what went wrong +// happened after the last one. +func (divergence Divergence) Matched() bool { + return divergence.StepIndex >= divergence.StepCount +} + +// Lines is everything the program printed since it last agreed with the +// transcript, in the order it printed it, ending with the line the report +// names as received. +// +// A single failure report picks one of these to put opposite the expected +// line, and the rest is context. A comparison cannot: which line of several +// was meant to be the step's is exactly what is in question -- the program +// that printed its own version of the line and carried on, and the program +// that printed three lines of error and died, both arrive here -- so it shows +// the block and lets it be read. +func (divergence Divergence) Lines() []string { + return append(append([]string{}, divergence.Context...), divergence.Received) +} + +// Outcome is how one run ended. Failure is nil when the run passed, and also +// when it could not be started at all -- a missing executable or an unreadable +// tape is a fault in the invocation rather than a divergence, and saying a +// port differs from the reference when it never ran would be a lie of exactly +// the kind this tool exists to catch. +type Outcome struct { + Name string + ExitCode int + Failure *Divergence +} + +// Report explains a divergence to whoever is watching, in a form meant to be +// read by a person or pasted into a bug report. +// +// Everything in it is either a fixed string, something the script says, or +// something the program printed. Nothing is measured: a timeout reports the +// limit it was given and not how long it actually waited, so two runs of the +// same divergence produce the same bytes and a real change stands out from a +// slow machine. +// +// Lines are quoted because trailing whitespace is load-bearing: prompts +// routinely end in a space, and an unquoted report makes a step that differs +// from another only in that respect look identical to it. +func (divergence Divergence) Report() string { + if divergence.Matched() { + return fmt.Sprintf("%s: all %d steps matched, but %s", + divergence.Mode, divergence.StepCount, divergence.Detail) + } + + var report strings.Builder + fmt.Fprintf(&report, "%s: step %d of %d did not match (%s)\n\n", + divergence.Mode, divergence.StepIndex+1, divergence.StepCount, divergence.Detail) + if divergence.Redacted { + fmt.Fprintf(&report, " expected %q (with redactions applied)\n", divergence.Expected) + } else { + fmt.Fprintf(&report, " expected %q\n", divergence.Expected) + } + fmt.Fprintf(&report, " received %q\n", divergence.Received) + // A character-level pointer only means something when the two are meant to + // be equal; against a pattern the first differing byte is noise. + if !divergence.Redacted { + report.WriteString(differenceMarker(divergence.Expected, divergence.Received, len(reportLabelIndent))) + } + + // The preceding lines are what makes a mismatch diagnosable when the + // received line is not the interesting one, e.g. the executable printed an + // error and exited before ever reaching the prompt. + if len(divergence.Context) > 0 { + report.WriteString("\n output since the last matched step:\n") + for _, line := range divergence.Context { + fmt.Fprintf(&report, " %s\n", line) + } + } + + if remaining := divergence.StepCount - divergence.StepIndex - 1; remaining > 0 { + fmt.Fprintf(&report, "\n%s never reached\n", pluralSteps(remaining)) + } + + return report.String() +} + +// reportLabelIndent is the prefix in front of the quoted expected/received +// lines. differenceMarker needs its width to line a caret up underneath. +const reportLabelIndent = " received " + +// differenceMarker points at the first character where the received line +// diverged from the expected one. Spotting that by eye in two quoted strings +// is exactly the tedium this tool exists to remove, and it is worst in the +// cases that matter most: a changed letter mid-word, or a trailing space. +func differenceMarker(expected, received string, indent int) string { + // With nothing received there is nothing to point at, and the empty quotes + // above already say it plainly. + if expected == received || received == "" { + return "" + } + + prefix := commonPrefix(expected, received) + label := "first difference" + if prefix == received { + label = "received output ends here" + } + + // The caret indexes the quoted rendering, so measure the prefix as it will + // appear once escaped. Quote adds a pair of quotes; only the opening one + // sits between the indent and the first differing character. + column := indent + len(strconv.Quote(prefix)) - 1 + return fmt.Sprintf("%s^ %s\n", strings.Repeat(" ", column), label) +} + +func commonPrefix(a, b string) string { + offset := 0 + for _, char := range a { + width := utf8.RuneLen(char) + if offset+width > len(b) || a[offset:offset+width] != b[offset:offset+width] { + break + } + offset += width + } + return a[:offset] +} + +func pluralSteps(count int) string { + if count == 1 { + return "1 later step was" + } + return fmt.Sprintf("%d later steps were", count) +} diff --git a/internal/play/play.go b/internal/play/play.go index 3564456..734ab18 100644 --- a/internal/play/play.go +++ b/internal/play/play.go @@ -110,22 +110,26 @@ func reportFailure(message string) { fmt.Fprintln(os.Stderr, message) } -func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { +// Run plays one run and says how it ended. A non-success code with no +// Failure means it never got as far as a step -- a missing executable, an +// unreadable tape -- and a run that did not happen is not a run that +// disagreed with anything. +func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) Outcome { executablePath, err := getExecutableFilePath(config, run) if err != nil { - return utils.USER_ERROR + return notPlayed(run, utils.USER_ERROR) } terminal, err := getTerminal(config, run) if err != nil { reportFailure(err.Error()) - return utils.USER_ERROR + return notPlayed(run, utils.USER_ERROR) } environment, err := childEnvironment(config, run) if err != nil { reportFailure(err.Error()) - return utils.USER_ERROR + return notPlayed(run, utils.USER_ERROR) } executable, err := utils.StartExecutable(utils.ExecutableOptions{ @@ -135,7 +139,13 @@ func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { Terminal: terminal, }, logger) if err != nil { - return utils.INTERNAL_ERROR + // Said out loud rather than logged away. The default log level is + // "none", so a run that could not be started otherwise exits non-zero + // having given no reason at all -- and in a comparison it is named as + // one that could not be played, which invites the question this + // answers. + reportFailure(fmt.Sprintf("could not start %s: %s", executablePath, err)) + return notPlayed(run, utils.INTERNAL_ERROR) } processor := NewOutputProcessor(executable.Stdout, terminal != utils.TerminalPipes, logger) @@ -145,7 +155,7 @@ func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { timeout := matcher.Timeout(config.Timeout) tokenResult := processor.NextToken(timeout) if tokenResult.Error != nil { - return reportInterrupted(&executable, &matcher, timeout, tokenResult.Error) + return reportInterrupted(run, &executable, &matcher, timeout, tokenResult.Error) } if tokenResult.Finished { @@ -164,14 +174,14 @@ func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { err = matcher.Match(char) if err != nil { - return utils.INTERNAL_ERROR + return notPlayed(run, utils.INTERNAL_ERROR) } } exitCode, err := executable.WaitForExit() if matcher.MissingSteps() { - reportFailure(matcher.FailureReport(ExitedEarly, fmt.Sprintf("the executable exited with %d", exitCode))) - return utils.CLI_ERROR + return diverged(run, utils.CLI_ERROR, + matcher.Divergence(ExitedEarly, fmt.Sprintf("the executable exited with %d", exitCode))) } // A non-zero exit arrives as an ExitError, and that is the program's answer @@ -181,19 +191,34 @@ func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { var exitError *exec.ExitError if err != nil && !errors.As(err, &exitError) { logger.Error("error waiting for process to finish: ", err) - return utils.INTERNAL_ERROR + return notPlayed(run, utils.INTERNAL_ERROR) } // we rely on exit code in the script to know whether to fail on errors if exitCode != run.ExitCode { - msg := matcher.FailureReport(WrongExitCode, + divergence := matcher.Divergence(WrongExitCode, fmt.Sprintf("the executable exited with %d and the script expects %d", exitCode, run.ExitCode)) - logger.Info(msg) - reportFailure(msg) - return utils.INTERNAL_ERROR + logger.Info(divergence.Report()) + return diverged(run, utils.INTERNAL_ERROR, divergence) } - return utils.SUCCESS + return Outcome{Name: run.Name, ExitCode: utils.SUCCESS} +} + +// notPlayed is a run that could not be attempted, or that failed prescript +// rather than the script. Whatever went wrong has already been said; there is +// no divergence to record because nothing was compared. +func notPlayed(run script.Run, code int) Outcome { + return Outcome{Name: run.Name, ExitCode: code} +} + +// diverged reports the divergence as it records it. Reporting here rather than +// at each call site is what keeps the two the same thing: a divergence that +// went into an outcome without being printed would be one a single-run script +// never mentions. +func diverged(run script.Run, code int, divergence Divergence) Outcome { + reportFailure(divergence.Report()) + return Outcome{Name: run.Name, ExitCode: code, Failure: &divergence} } // reportInterrupted deals with a run that stopped while the program was still @@ -209,17 +234,18 @@ func Run(config cfg.PlayConfig, run script.Run, logger utils.Logger) int { // at; a program that matched every step and then would not exit is waiting for // something nobody is going to send. Those are different bugs and they are // named differently. -func reportInterrupted(executable *utils.Executable, matcher *StepMatcher, timeout time.Duration, err error) int { +func reportInterrupted(run script.Run, executable *utils.Executable, matcher *StepMatcher, timeout time.Duration, err error) Outcome { executable.Terminate() switch { case !errors.Is(err, ErrTimeout): - reportFailure(matcher.FailureReport(ReadFailed, fmt.Sprintf("could not read from the executable: %s", err))) + return diverged(run, utils.CLI_ERROR, + matcher.Divergence(ReadFailed, fmt.Sprintf("could not read from the executable: %s", err))) case matcher.MissingSteps(): - reportFailure(matcher.FailureReport(NoMatch, fmt.Sprintf("nothing matched it within %s", timeout))) + return diverged(run, utils.CLI_ERROR, + matcher.Divergence(NoMatch, fmt.Sprintf("nothing matched it within %s", timeout))) default: - reportFailure(matcher.FailureReport(Hung, fmt.Sprintf("the executable had not exited after %s", timeout))) + return diverged(run, utils.CLI_ERROR, + matcher.Divergence(Hung, fmt.Sprintf("the executable had not exited after %s", timeout))) } - - return utils.CLI_ERROR } diff --git a/internal/play/play_integration_test.go b/internal/play/play_integration_test.go index d5f0b2d..f0ce960 100644 --- a/internal/play/play_integration_test.go +++ b/internal/play/play_integration_test.go @@ -48,14 +48,14 @@ func createConfig(t *testing.T, fileName string) cfg.Config { func TestOutput(t *testing.T) { t.Parallel() config := createConfig(t, "fixtures/output.sh") - exitCode := Run(config.Play, script.Run{}, config.Logger) + exitCode := Run(config.Play, script.Run{}, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } func TestOutputWithDelay(t *testing.T) { t.Parallel() config := createConfig(t, "fixtures/output_with_delay.sh") - exitCode := Run(config.Play, script.Run{}, config.Logger) + exitCode := Run(config.Play, script.Run{}, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -65,7 +65,7 @@ func TestInput(t *testing.T) { config := createConfig(t, "fixtures/input.sh") run := script.Run{Steps: []script.Step{{Line: "Please enter your name: ", Input: "Richard"}}} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -79,7 +79,7 @@ func TestDoubleInput(t *testing.T) { {Line: "Second number: ", Input: "2"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -95,7 +95,7 @@ func TestDoubleInputRedaction(t *testing.T) { {Line: "Sum: {{digit}}", LineRegex: *regex, Redacted: true}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -109,7 +109,7 @@ func TestPassingArguments(t *testing.T) { Arguments: []string{"Rachel"}, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -125,7 +125,7 @@ func TestSpecifyExecutableInScript(t *testing.T) { } config.Play.ExecutablePath = "" - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -146,7 +146,7 @@ func TestDuplicatedLinesInScript(t *testing.T) { {Line: "success!"}, }, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -162,7 +162,7 @@ func TestCheckExitCode(t *testing.T) { }, ExitCode: 1, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -176,7 +176,7 @@ func TestFailIfUnrecognisedStep(t *testing.T) { {Line: "Hello, Rachel"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 1, exitCode) } @@ -190,7 +190,7 @@ func TestFailIfUnexpectedStdin(t *testing.T) { {Line: "Hello, Rachel"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 1, exitCode) } @@ -206,7 +206,7 @@ func TestFailIfExecutableTimesOutAfterSteps(t *testing.T) { {Line: "Expecting this line"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 1, exitCode) } @@ -227,7 +227,7 @@ func TestDeclaredEnvReplacesTheParentEnvironment(t *testing.T) { }, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -245,7 +245,7 @@ func TestInheritEnvKeepsTheParentEnvironment(t *testing.T) { }, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -259,7 +259,7 @@ func TestNoDeclaredEnvInheritsTheParentEnvironment(t *testing.T) { {Line: "PARENT=visible"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -279,7 +279,7 @@ func TestRedactionMatchesAVolatileValue(t *testing.T) { parsed, err := script.ParseScriptFromBytes([]byte(document)) assert.NoError(t, err) - exitCode := Run(config.Play, parsed.Runs[0], config.Logger) + exitCode := Run(config.Play, parsed.Runs[0], config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -296,7 +296,7 @@ func TestRedactionStillFailsOnARealMismatch(t *testing.T) { parsed, err := script.ParseScriptFromBytes([]byte(document)) assert.NoError(t, err) - exitCode := Run(config.Play, parsed.Runs[0], config.Logger) + exitCode := Run(config.Play, parsed.Runs[0], config.Logger).ExitCode assert.NotEqual(t, 0, exitCode) } @@ -314,7 +314,7 @@ func TestExecutableRunsUnderAPtyByDefault(t *testing.T) { {Line: "Hello, Rachel"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -329,7 +329,7 @@ func TestThePtyIsAlwaysTheSameSize(t *testing.T) { config := createConfig(t, "fixtures/terminal_size.sh") run := script.Run{Steps: []script.Step{{Line: "24 80"}}} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -345,7 +345,7 @@ func TestTerminalPipesGivesTheExecutablePipes(t *testing.T) { {Line: "Hello, Rachel"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -363,7 +363,7 @@ func TestRunCanAskForPipes(t *testing.T) { }, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -384,7 +384,7 @@ func TestCommandLineTerminalOverridesTheScript(t *testing.T) { }, } - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -395,7 +395,7 @@ func TestUnrecognisedTerminalIsRejected(t *testing.T) { config := createConfig(t, "fixtures/tty.sh") config.Play.Terminal = "tty" - exitCode := Run(config.Play, script.Run{}, config.Logger) + exitCode := Run(config.Play, script.Run{}, config.Logger).ExitCode assert.Equal(t, utils.USER_ERROR, exitCode) } @@ -414,7 +414,7 @@ func TestInputIsNotEchoedBackUnderAPty(t *testing.T) { {Line: "How do you do, Richard"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -451,7 +451,7 @@ func TestFailureModesAreNamed(t *testing.T) { config := createConfig(t, "fixtures/input.sh") config.Play.Timeout = getTimeout(500) run := script.Run{Steps: []script.Step{{Line: "Hello, Rachel"}}} - assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger)) + assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger).ExitCode) }) assert.Contains(t, report, "no-match: step 1 of 1 did not match (nothing matched it within 500ms)") @@ -459,7 +459,7 @@ func TestFailureModesAreNamed(t *testing.T) { report = captureStderr(t, func() { config := createConfig(t, "fixtures/output.sh") run := script.Run{Steps: []script.Step{{Line: "Hello, Rachel"}}} - assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger)) + assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger).ExitCode) }) assert.Contains(t, report, "exited-early: step 1 of 1 did not match (the executable exited with 0)") @@ -468,7 +468,7 @@ func TestFailureModesAreNamed(t *testing.T) { config := createConfig(t, "fixtures/timeout.sh") config.Play.Timeout = getTimeout(500) run := script.Run{Steps: []script.Step{{Line: "Expecting this line"}}} - assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger)) + assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger).ExitCode) }) assert.Equal(t, "hung: all 1 steps matched, but the executable had not exited after 500ms\n", report) @@ -476,7 +476,7 @@ func TestFailureModesAreNamed(t *testing.T) { report = captureStderr(t, func() { config := createConfig(t, "fixtures/exit_code.sh") run := script.Run{Steps: []script.Step{{Line: "Exit Code test"}}} - assert.Equal(t, utils.INTERNAL_ERROR, Run(config.Play, run, config.Logger)) + assert.Equal(t, utils.INTERNAL_ERROR, Run(config.Play, run, config.Logger).ExitCode) }) assert.Contains(t, report, "wrong-exit-code: all 1 steps matched, but the executable exited with 1 and the script expects 0") } @@ -493,7 +493,7 @@ func TestStepTimeoutOverridesTheRunTimeout(t *testing.T) { {Line: "Done", Timeout: "5s", TimeoutDuration: 5 * time.Second}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, 0, exitCode) } @@ -510,7 +510,7 @@ func TestSlowStepFailsWithoutAnOverride(t *testing.T) { {Line: "Done"}, }} - exitCode := Run(config.Play, run, config.Logger) + exitCode := Run(config.Play, run, config.Logger).ExitCode assert.Equal(t, utils.CLI_ERROR, exitCode) } @@ -528,7 +528,7 @@ func TestTimeoutKillsTheExecutableAndItsChildren(t *testing.T) { Steps: []script.Step{{Line: "Started"}}, } - assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger)) + assert.Equal(t, utils.CLI_ERROR, Run(config.Play, run, config.Logger).ExitCode) // The background process writes the marker a second in. Long enough after // that to be sure: it either never ran again, or it did. diff --git a/internal/play/runner_integration_test.go b/internal/play/runner_integration_test.go index 8dd7217..639d455 100644 --- a/internal/play/runner_integration_test.go +++ b/internal/play/runner_integration_test.go @@ -98,11 +98,11 @@ func TestRunnersMakeRandomnessRepeatable(t *testing.T) { runs := script.ApplyRunner([]script.Run{drawingRun(t, language.program, language.drawn)}, runner) logger := &utils.CustomLogger{} - assert.Equal(t, 0, Run(drawingConfig(), runs[0], logger)) + assert.Equal(t, 0, Run(drawingConfig(), runs[0], logger).ExitCode) // Twice, because "deterministic" is a claim about runs and not // about one run. - assert.Equal(t, 0, Run(drawingConfig(), runs[0], logger)) + assert.Equal(t, 0, Run(drawingConfig(), runs[0], logger).ExitCode) }) } } @@ -123,7 +123,7 @@ func TestWithoutARunnerTheSameProgramIsUnscriptable(t *testing.T) { exitCode := 0 report := captureStderr(t, func() { - exitCode = Run(drawingConfig(), run, &utils.CustomLogger{}) + exitCode = Run(drawingConfig(), run, &utils.CustomLogger{}).ExitCode }) assert.NotEqual(t, 0, exitCode, "expected unseeded output to differ from the recorded draw") @@ -173,7 +173,7 @@ func TestOneTapeGivesEveryLanguageTheSameNumbers(t *testing.T) { runner := loadRunner(t, language.runner) runs := script.ApplyRunner([]script.Run{drawingRun(t, language.program, referenceDraws)}, runner) - assert.Equal(t, 0, Run(tapedConfig(t), runs[0], &utils.CustomLogger{})) + assert.Equal(t, 0, Run(tapedConfig(t), runs[0], &utils.CustomLogger{}).ExitCode) }) } } @@ -224,7 +224,7 @@ func TestRunningOffTheEndOfTheTapeFails(t *testing.T) { exitCode := 0 report := captureStderr(t, func() { - exitCode = Run(config, runs[0], &utils.CustomLogger{}) + exitCode = Run(config, runs[0], &utils.CustomLogger{}).ExitCode }) assert.NotEqual(t, 0, exitCode) @@ -245,7 +245,7 @@ func TestAPortRecordsHowMuchOfTheTapeItUsed(t *testing.T) { usage := filepath.Join(t.TempDir(), "usage") runs[0].Env["PRESCRIPT_TAPE_USAGE"] = usage - assert.Equal(t, 0, Run(tapedConfig(t), runs[0], &utils.CustomLogger{})) + assert.Equal(t, 0, Run(tapedConfig(t), runs[0], &utils.CustomLogger{}).ExitCode) drawn, err := os.ReadFile(usage) assert.NoError(t, err) @@ -266,7 +266,7 @@ func TestAMissingTapeIsReportedBeforeTheRun(t *testing.T) { exitCode := 0 report := captureStderr(t, func() { - exitCode = Run(config, run, &utils.CustomLogger{}) + exitCode = Run(config, run, &utils.CustomLogger{}).ExitCode }) assert.Equal(t, utils.USER_ERROR, exitCode) diff --git a/internal/play/runs.go b/internal/play/runs.go index c428239..8108d3a 100644 --- a/internal/play/runs.go +++ b/internal/play/runs.go @@ -3,8 +3,6 @@ package play import ( "fmt" "os" - "strconv" - "strings" cfg "github.com/rjnienaber/prescript/internal/config" "github.com/rjnienaber/prescript/internal/script" @@ -25,7 +23,7 @@ func RunAll(config cfg.PlayConfig, runs []script.Run, logger utils.Logger) int { } result := utils.SUCCESS - var failed []string + outcomes := make([]Outcome, 0, len(runs)) for _, run := range runs { if multiple { @@ -36,19 +34,20 @@ func RunAll(config cfg.PlayConfig, runs []script.Run, logger utils.Logger) int { fmt.Fprintf(os.Stderr, "\n=== %s ===\n", run.Name) } - code := Run(config, run, logger) - if code == utils.SUCCESS { - continue - } - - failed = append(failed, strconv.Quote(run.Name)) - if result == utils.SUCCESS { - result = code + outcome := Run(config, run, logger) + outcomes = append(outcomes, outcome) + if outcome.ExitCode != utils.SUCCESS && result == utils.SUCCESS { + result = outcome.ExitCode } } - if multiple && len(failed) > 0 { - fmt.Fprintf(os.Stderr, "\n%d of %d runs failed: %s\n", len(failed), len(runs), strings.Join(failed, ", ")) + // The comparison rather than a tally of how many runs failed. It says + // everything the tally did -- a port that diverged is named, and so is one + // that never ran -- and says it aligned by the step where each port left + // the reference, which is the thing a multi-run script was written to + // find out. + if multiple { + fmt.Fprint(os.Stderr, Compare(runs, outcomes).Report()) } return result diff --git a/internal/play/step_matcher.go b/internal/play/step_matcher.go index 9dace6b..46dd872 100644 --- a/internal/play/step_matcher.go +++ b/internal/play/step_matcher.go @@ -3,10 +3,7 @@ package play import ( "fmt" "io" - "strconv" - "strings" "time" - "unicode/utf8" "github.com/rjnienaber/prescript/internal/script" "github.com/rjnienaber/prescript/internal/utils" @@ -95,58 +92,34 @@ func (matcher *StepMatcher) receivedLine() string { return "" } -// FailureReport explains why a run stopped short, in a form meant to be read -// by a person or pasted into a bug report. mode names which of the ways it -// went wrong, and detail says what was observed, for example "the executable -// exited with 1". -// -// Everything in a report is either a fixed string, something the script says, -// or something the program printed. Nothing is measured: a timeout reports the -// limit it was given and not how long it actually waited, so two runs of the -// same divergence produce the same bytes and a real change stands out from a -// slow machine. -// -// Lines are quoted because trailing whitespace is load-bearing: prompts -// routinely end in a space, and an unquoted report makes a step that differs -// only in that respect look identical to the one it failed to match. -func (matcher *StepMatcher) FailureReport(mode FailureMode, detail string) string { - if !matcher.MissingSteps() { - return fmt.Sprintf("%s: all %d steps matched, but %s", mode, len(matcher.steps), detail) - } - - step := matcher.steps[matcher.currentStepIndex] - expected, received := matcher.NextExpectedLine(), matcher.receivedLine() - - var report strings.Builder - fmt.Fprintf(&report, "%s: step %d of %d did not match (%s)\n\n", - mode, matcher.currentStepIndex+1, len(matcher.steps), detail) - if step.Redacted { - fmt.Fprintf(&report, " expected %q (with redactions applied)\n", expected) - } else { - fmt.Fprintf(&report, " expected %q\n", expected) - } - fmt.Fprintf(&report, " received %q\n", received) - // A character-level pointer only means something when the two are meant to - // be equal; against a pattern the first differing byte is noise. - if !step.Redacted { - report.WriteString(differenceMarker(expected, received, len(reportLabelIndent))) +// Divergence records where this run stopped agreeing with its transcript. +// mode names which of the ways it went wrong, and detail says what was +// observed, for example "the executable exited with 1". +func (matcher *StepMatcher) Divergence(mode FailureMode, detail string) Divergence { + divergence := Divergence{ + Mode: mode, + Detail: detail, + StepIndex: matcher.currentStepIndex, + StepCount: len(matcher.steps), } - // The preceding lines are what makes a mismatch diagnosable when the - // received line is not the interesting one, e.g. the executable printed an - // error and exited before ever reaching the prompt. - if context := matcher.contextLines(); len(context) > 0 { - report.WriteString("\n output since the last matched step:\n") - for _, line := range context { - fmt.Fprintf(&report, " %s\n", line) - } + if !matcher.MissingSteps() { + return divergence } - if remaining := len(matcher.steps) - matcher.currentStepIndex - 1; remaining > 0 { - fmt.Fprintf(&report, "\n%s never reached\n", pluralSteps(remaining)) - } + step := matcher.steps[matcher.currentStepIndex] + divergence.Expected = step.Line + divergence.Received = matcher.receivedLine() + divergence.Redacted = step.Redacted + divergence.Context = matcher.contextLines() + return divergence +} - return report.String() +// FailureReport is the divergence written out. Kept as one call because the +// overwhelmingly common case -- a single run, reported to stderr and read by a +// person -- has no use for the value in between. +func (matcher *StepMatcher) FailureReport(mode FailureMode, detail string) string { + return matcher.Divergence(mode, detail).Report() } // contextLines are the completed lines leading up to the failure, excluding the @@ -159,53 +132,6 @@ func (matcher *StepMatcher) contextLines() []string { return lines } -// reportLabelIndent is the prefix in front of the quoted expected/received -// lines. differenceMarker needs its width to line a caret up underneath. -const reportLabelIndent = " received " - -// differenceMarker points at the first character where the received line -// diverged from the expected one. Spotting that by eye in two quoted strings -// is exactly the tedium this tool exists to remove, and it is worst in the -// cases that matter most: a changed letter mid-word, or a trailing space. -func differenceMarker(expected, received string, indent int) string { - // With nothing received there is nothing to point at, and the empty quotes - // above already say it plainly. - if expected == received || received == "" { - return "" - } - - prefix := commonPrefix(expected, received) - label := "first difference" - if prefix == received { - label = "received output ends here" - } - - // The caret indexes the quoted rendering, so measure the prefix as it will - // appear once escaped. Quote adds a pair of quotes; only the opening one - // sits between the indent and the first differing character. - column := indent + len(strconv.Quote(prefix)) - 1 - return fmt.Sprintf("%s^ %s\n", strings.Repeat(" ", column), label) -} - -func commonPrefix(a, b string) string { - offset := 0 - for _, char := range a { - width := utf8.RuneLen(char) - if offset+width > len(b) || a[offset:offset+width] != b[offset:offset+width] { - break - } - offset += width - } - return a[:offset] -} - -func pluralSteps(count int) string { - if count == 1 { - return "1 later step was" - } - return fmt.Sprintf("%d later steps were", count) -} - func (matcher *StepMatcher) Match(char string) error { matcher.currentLine += char if matcher.currentStepIndex < len(matcher.steps) {