From 40a961a0c5f1d8d729cf27d57b16417500b90e49 Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 22:54:43 +0100 Subject: [PATCH 1/4] generate ready-to-file bug reports The usual answer to a cross-implementation bug report is "works for me", and it is usually fair: the two people ran different interpreter builds, in different locales, against different random numbers. Pinning all of that down by hand is ten minutes of copying version strings out of terminals, so nobody does it twice and by the tenth report they are not being filed at all. --bug-report writes the first divergence to a file as Markdown, with everything the run already knew about itself: both sides and the classification, the interpreter version, the container digest, the tape and its provenance plus how much of it the port drew, LC_ALL/LANG/TZ, the terminal and its size, prescript's own revision, and one command that reproduces the lot. The first divergence only. A run that went wrong at step 5 was not really running the same program from step 6 onwards, and listing every later difference asks the reader to work out which one was the cause. Two facts prescript could not previously state about itself had to exist first: TerminalSize(), moved out of the unix-tagged pty file, and Version() from the binary's own build info -- which also required the Makefile to build a package path rather than a file path, since that is the only form go build stamps VCS info onto. --version then needed Config.Handled so cobra answering the command line itself exits zero rather than as a usage error. Only an executable named by name is asked for its version. One named by a path is the program under test rather than the toolchain under it. Only a registry digest earns a `make pull_image` line; a locally built image's config digest is named but not pasted as a command nobody can run. Absolute paths inside the working directory are made relative, so a report can be posted in public without a home directory in it. Closes #18 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- Makefile | 4 +- README.md | 19 + cmd/prescript/main.go | 10 +- docs/bug-reports.md | 179 +++++++++ docs/comparison.md | 6 + internal/config/play.go | 2 + internal/config/root.go | 13 + internal/play/bug_report.go | 446 +++++++++++++++++++++++ internal/play/bug_report_test.go | 260 +++++++++++++ internal/play/runner_integration_test.go | 37 ++ internal/play/runs.go | 24 ++ internal/utils/pty_unix.go | 14 - internal/utils/terminal_size.go | 24 ++ internal/utils/version.go | 57 +++ 14 files changed, 1078 insertions(+), 17 deletions(-) create mode 100644 docs/bug-reports.md create mode 100644 internal/play/bug_report.go create mode 100644 internal/play/bug_report_test.go create mode 100644 internal/utils/terminal_size.go create mode 100644 internal/utils/version.go diff --git a/Makefile b/Makefile index f605d35..2c7ad11 100644 --- a/Makefile +++ b/Makefile @@ -94,11 +94,11 @@ lint: .PHONY: build_dev build_dev: - go build -o tmp/prescript cmd/prescript/main.go + go build -o tmp/prescript ./cmd/prescript .PHONY: build_release build_release: - go build -ldflags="-s -w" -a -o tmp/prescript cmd/prescript/main.go + go build -ldflags="-s -w" -a -o tmp/prescript ./cmd/prescript .PHONY: test test: diff --git a/README.md b/README.md index e0aaa32..61061ca 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ prescript play [script file] [flags] -- [executable arguments] | `-t` | timeout waiting for output from external command | `bool` | `30s ` | No | | `--terminal` | what the executable is given for its standard streams (`pty` or `pipes`) | `enum` | `pty` | No | | `--tape` | file of recorded random values to replay to the executable | `string` | | No | +| `--bug-report` | write the first divergence to this file as a report someone else can act on | `string` | | No | | `-- [args]` | arguments for the executable, overriding those in the script | `list` | | No | #### `record` @@ -298,6 +299,24 @@ one report per language rather than one per program. `no-draws` means the port had taken nothing from the tape, so randomness is not involved. See [docs/comparison.md](docs/comparison.md). +### Filing what it found + +The usual answer to a cross-implementation bug report is "works for me", and it +is usually fair: the two people ran different interpreter builds, in different +locales, against different random numbers. `--bug-report` writes the first +divergence to a file with all of that pinned down, because the run already knew +it: + +```bash +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +The file holds both sides of the divergence and its classification, the +interpreter version, the container digest, the tape and how much of it the port +drew, the locale, timezone and terminal size, prescript's own revision, and one +command that reproduces the lot. It files nothing and changes no exit code. See +[docs/bug-reports.md](docs/bug-reports.md). + ### Development ``` diff --git a/cmd/prescript/main.go b/cmd/prescript/main.go index 3b55bf2..2b7fd09 100644 --- a/cmd/prescript/main.go +++ b/cmd/prescript/main.go @@ -14,7 +14,15 @@ import ( func main() { config, err := cfg.GetConfig() - if err != nil || config.Subcommand == cfg.NoCommand { + if err != nil { + os.Exit(utils.USER_ERROR) + } + + if config.Handled { + os.Exit(utils.SUCCESS) + } + + if config.Subcommand == cfg.NoCommand { os.Exit(utils.USER_ERROR) } diff --git a/docs/bug-reports.md b/docs/bug-reports.md new file mode 100644 index 0000000..4e0272f --- /dev/null +++ b/docs/bug-reports.md @@ -0,0 +1,179 @@ +# Bug reports + +``` +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +`--bug-report` writes the first divergence to a file, as Markdown, with +everything somebody else needs to reproduce it. + +## Why prescript writes it rather than you + +The default answer to a cross-implementation bug report is "works for me", and +it is usually a fair answer. Two people running the same program in two +languages are also running two interpreter builds, two locales, and two +sequences of random numbers, and any one of those explains a difference on its +own. A report that does not pin all of them down cannot be acted on, only +argued with. + +Pinning them down by hand is perhaps ten minutes of copying version strings out +of terminals. Nobody does it twice, so the second report is worse than the +first, and by the tenth the reports have stopped being filed at all. Everything +in the block below is something the run already knew about itself, so it is +written down rather than remembered. + +## What it writes + +````markdown +# ruby: no-match at step 5 of 12 + +In `dice.yaml`, against `basic`. +Classified `formatting, same-draws`. + +## The divergence + +``` +step 5 of 12 (no-match, formatting, same-draws) + basic " 7 807 " + ruby "7 807" + ^ first difference +``` + +This is the first divergence; 7 later steps were never reached. + +## Reproducing it + +```sh +make pull_image PUBLISHED_REF=ghcr.io/rjnienaber/prescript-env@sha256:2a1f... +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +## The environment + +| | | +| --- | --- | +| prescript | `86fa312b39df` | +| script | `dice.yaml` | +| command | `ruby ports/ruby/dice.rb` | +| ruby | `ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-linux]` | +| image | `ghcr.io/rjnienaber/prescript-env@sha256:2a1f...` | +| tape | `tapes/vintbas-seed0.tape` | +| draws | `5` | +| terminal | `pty, 80x24` | +| timeout | `30s` | +| LC_ALL | `C.UTF-8` | +| TZ | `UTC` | + +The tape says of itself: + +``` +# prescript tape 1 +# source: vintage-basic 1.0.3, built as vintbas-1.0.3-1 +# random: 1.2.1.2 +# seed: 0 +# values: 10000 +``` +```` + +The three sections are in the order they get read in. Somebody who already +knows the bug stops after the first; somebody about to say "works for me" is +looking for the third. + +## The first divergence only + +A run that went wrong at step 5 was not really running the same program from +step 6 onwards. Everything after the first difference is a consequence of it, +and a report that lists all of them is asking its reader to work out which one +was the cause — which is the work the report was supposed to do. + +Which divergence that is depends on what was played: + +- **A script with several runs.** The earliest finding in the + [comparison](comparison.md): the first step at which any port left the + reference, and every port that left it there in the same way. Findings are + grouped, so a report about one bug in twelve ports is one report naming + twelve ports. +- **A script whose reference failed.** The reference's own divergence. That is + the more urgent finding rather than a reason to write nothing: the + environment has moved under a script that used to pass, and the ports were + never compared against anything. +- **A single run.** Its own failure, reported against `the transcript`. There is + no implementation being treated as correct, and naming one that was never + played would misdescribe where the expected side came from. + +When every run matched there is nothing to write, and prescript says so instead +of writing an empty file. + +## What is in the facts table, and why + +Each row is there because it has been the answer to a "works for me" at least +once. + +| Row | Why it is there | +| --- | --- | +| `prescript` | The revision that produced the report, from the binary's own build info. Reports outlive releases. | +| `script` | Which script, since one corpus has hundreds. | +| `command` | The executable and arguments the run actually used, after any runner and any `--exec`. | +| the interpreter | What the toolchain says it is. See below. | +| `image` | The pinned digest, which is what makes everything else reproducible. | +| `tape` or `seed` | Where the randomness came from. Without it the report is not reproducible even in principle. | +| `draws` | How much of the tape the port had used when it stopped. Zero means randomness is not implicated at all. | +| `terminal` | `pty, 80x24` or `pipes`. Programs lay their output out differently depending on the answer, and a size is not something a reader can guess. | +| `timeout` | What a timeout meant on this run. | +| `LC_ALL`, `LANG`, `TZ` | The three variables most likely to change a program's output between two machines that are otherwise identical. | + +The locale and timezone are read from the environment the program was actually +started with, which is the run's own when it +[declared one](script-format.md) and prescript's otherwise. + +### Asking the interpreter what it is + +The executable is run once more with `--version` (`version` for `go`), and the +first line of its output becomes a row. Only an executable named by name is +asked. One named by a path — `./dice`, `ports/ruby/dice.rb` — is the program +under test rather than the toolchain under it, and running somebody else's +program with an argument it never expected is not a thing to do behind their +back. The probe is bounded at five seconds and omitted entirely if it fails, so +it cannot become a second way for a failed run to hang. + +### The image + +A run outside the [pinned container](container.md) says so, in the table, in +those words: + +``` +| image | `none; this did not run in the pinned environment` | +``` + +That is a fact worth stating rather than a gap worth leaving. It is the single +likeliest reason the reader cannot reproduce what they are being shown. + +The `make pull_image` line appears in the repro block only when the digest is +one somebody else can pull. A locally built image is identified by its config +digest — a bare `sha256:` with no name in front of it — which says which bytes +ran but cannot be fetched, so it is named in the table and left out of the +commands. + +## The repro command + +The command line is reproduced as it was given, with two changes: + +- `argv[0]` becomes `prescript`. How this machine reaches the binary is not + part of the bug. +- Absolute paths inside the working directory become relative to it. This makes + the command work from a fresh checkout, and it keeps a home directory out of + a report that is about to be posted in public. + +Paths outside the working directory are left alone: shortening them would +produce a command that does not work. If a report is going somewhere public, +this is the line to read before pasting it. + +## What it does not do + +It does not file anything. The output is a file; where it goes is a decision +with a person in it. + +It does not change the exit code. Whether the runs passed was decided by the +runs. A report that could not be written is said out loud on stderr — the run +it described has already finished, and will not be repeated for free — but it +is not itself a failure of the run. diff --git a/docs/comparison.md b/docs/comparison.md index 01ce449..bc2acd5 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -186,3 +186,9 @@ different output belong in two scripts. `prescript` still exits with the first non-zero code any run produced. The comparison is a report, not a verdict: it says where the implementations parted company, and what to do about that is the reader's call. + +## Filing a finding + +`--bug-report ` writes the earliest finding out as Markdown, with the +interpreter version, the container digest, the tape and everything else needed +to answer "works for me". See [bug-reports.md](bug-reports.md). diff --git a/internal/config/play.go b/internal/config/play.go index 39d6641..948ba49 100644 --- a/internal/config/play.go +++ b/internal/config/play.go @@ -16,6 +16,7 @@ type PlayConfig struct { RunnerFile string Terminal string Tape string + BugReport string Arguments []string } @@ -39,6 +40,7 @@ func createPlaySubCommand(config *Config) *cobra.Command { playCmd.Flags().StringVarP(&config.Play.RunnerFile, "runner", "r", "", "runner file describing how to launch an implementation") playCmd.Flags().StringVar(&config.Play.Terminal, "terminal", "", "override what the executable is given for its standard streams (pty or pipes)") playCmd.Flags().StringVar(&config.Play.Tape, "tape", "", "file of recorded random values to replay to the executable, instead of letting it draw its own") + playCmd.Flags().StringVar(&config.Play.BugReport, "bug-report", "", "write the first divergence to this file as a report someone else can act on") defaultTimeout := 30 * time.Second playCmd.Flags().DurationVarP(&config.Play.Timeout, "timeout", "t", defaultTimeout, "timeout waiting for output from external command") diff --git a/internal/config/root.go b/internal/config/root.go index 23ef516..8113346 100644 --- a/internal/config/root.go +++ b/internal/config/root.go @@ -14,6 +14,12 @@ const ( ) type Config struct { + // Handled is set when cobra has already answered the command line itself + // -- `--version` prints and there is nothing left to run. It is not a + // failure, and a caller that treats "no subcommand" as a usage error would + // otherwise exit non-zero having done exactly what was asked. + Handled bool + Subcommand subcommand Play PlayConfig Record RecordConfig @@ -30,6 +36,9 @@ func GetConfig() (Config, error) { Use: "prescript [cmd] [script file] [optional executable]", Short: "Record and playback responses to an interactive cli", Long: "Record and playback responses to an interactive cli", + // The same string a generated bug report names itself by, so someone + // reading one can check what they have against what made it. + Version: utils.Version(), } rootCmd.AddCommand(playCmd) @@ -40,5 +49,9 @@ func GetConfig() (Config, error) { return Config{}, err } + if asked, _ := rootCmd.Flags().GetBool("version"); asked { + config.Handled = true + } + return config, nil } diff --git a/internal/play/bug_report.go b/internal/play/bug_report.go new file mode 100644 index 0000000..cf1fc7d --- /dev/null +++ b/internal/play/bug_report.go @@ -0,0 +1,446 @@ +package play + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + cfg "github.com/rjnienaber/prescript/internal/config" + "github.com/rjnienaber/prescript/internal/script" + "github.com/rjnienaber/prescript/internal/utils" +) + +// BugReport is one divergence written up so that somebody else can act on it +// without re-deriving the setup. +// +// The default answer to any cross-implementation bug report is "works for me", +// and it is usually a fair answer: the two people ran different interpreter +// builds, in different locales, against different random numbers. Only a +// complete repro block prevents it, and assembling one by hand for every +// finding is work nobody does twice. So it is assembled here, from what the +// run already knows about itself. +// +// The first divergence only. A run that went wrong at step 5 was not really +// running the program from step 6 onwards, and everything after the first +// difference is a consequence of it. +type BugReport struct { + Script string + Reference string + Finding DivergenceGroup + + // Command reproduces the run. It is the command line prescript was given, + // which is the honest answer to "what did you run" -- rewritten only to + // drop the path prescript itself was invoked by and to make repository + // paths relative, so a report can be pasted somewhere public without + // carrying a home directory into it. + Command []string + + // Toolchain is what the pinned environment says about itself: the base + // image digest and the version of every interpreter in it, written at + // build time from the binaries actually installed. Empty outside it. + Toolchain string + + // TapeHeader is the provenance the tape carries: which interpreter + // recorded it, against which generator, from which seed. + TapeHeader []string + + Facts []Fact +} + +// Fact is one row of the environment table. Ordered, because the order is the +// argument: what was run, then what it was run against, then what was around +// it. +type Fact struct { + Name string + Value string +} + +// NewBugReport picks the divergence worth reporting and gathers everything +// around it. It reports false when there is nothing to report -- every run +// passed, or the runs that failed never got as far as diverging. +func NewBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome, commandLine []string) (BugReport, bool) { + reference, finding, found := firstFinding(runs, outcomes) + if !found { + return BugReport{}, false + } + + run, outcome := runFor(runs, outcomes, finding.Ports[0]) + report := BugReport{ + Script: config.ScriptFile, + Reference: reference, + Finding: finding, + Command: reproCommand(commandLine), + Toolchain: readFile(os.Getenv("PRESCRIPT_ENVIRONMENT")), + TapeHeader: tapeHeader(config.Tape), + } + report.Facts = gatherFacts(config, run, outcome) + return report, true +} + +// firstFinding is the divergence to report: the earliest one in a comparison, +// or the single run's own when there is nothing to compare it against. +func firstFinding(runs []script.Run, outcomes []Outcome) (string, DivergenceGroup, bool) { + if len(outcomes) > 1 { + comparison := Compare(runs, outcomes) + if len(comparison.Groups) > 0 { + return comparison.Reference, comparison.Groups[0], true + } + + // A reference that does not match its own transcript is still a + // finding, and often the more urgent one: the environment has moved + // under a script that used to pass. + if comparison.ReferenceFailure != nil { + return transcriptLabel, asGroup(comparison.Reference, *comparison.ReferenceFailure), true + } + return "", DivergenceGroup{}, false + } + + if len(outcomes) == 1 && outcomes[0].Failure != nil { + return transcriptLabel, asGroup(outcomes[0].Name, *outcomes[0].Failure), true + } + return "", DivergenceGroup{}, false +} + +// asGroup makes one run's divergence look like a finding of one, so a report +// about a single run and a report about a comparison are the same report. +// There is no reference implementation on this path, so the expected line +// speaks for itself and is labelled as what the script asked for. +func asGroup(name string, divergence Divergence) DivergenceGroup { + group := DivergenceGroup{ + StepIndex: divergence.StepIndex, + StepCount: divergence.StepCount, + Mode: divergence.Mode, + Expected: divergence.Expected, + Redacted: divergence.Redacted, + Received: divergence.Lines(), + Matched: divergence.Matched(), + Ports: []string{name}, + } + if group.Matched { + group.Received = []string{divergence.Detail} + } + group.Classification = group.classify(nil, []*int{nil}) + return group +} + +func runFor(runs []script.Run, outcomes []Outcome, name string) (script.Run, Outcome) { + for index, outcome := range outcomes { + if outcome.Name == name { + return runs[index], outcome + } + } + return script.Run{}, Outcome{} +} + +func gatherFacts(config cfg.PlayConfig, run script.Run, outcome Outcome) []Fact { + facts := []Fact{ + {"prescript", utils.Version()}, + {"script", config.ScriptFile}, + {"command", strings.TrimSpace(run.Executable + " " + strings.Join(run.Arguments, " "))}, + } + + if toolchain := toolchainVersion(run.Executable); toolchain != "" { + facts = append(facts, Fact{run.Executable, toolchain}) + } + + // An unpinned run is worth saying out loud rather than leaving as a gap. + // It is the single likeliest reason a reader cannot reproduce what they + // are being shown. + if image := os.Getenv("PRESCRIPT_IMAGE"); image != "" { + facts = append(facts, Fact{"image", image}) + } else { + facts = append(facts, Fact{"image", "none; this did not run in the pinned environment"}) + } + + environment := childEnv(run) + if config.Tape != "" { + facts = append(facts, Fact{"tape", config.Tape}) + } else if seed := lookup(environment, "PRESCRIPT_SEED"); seed != "" { + facts = append(facts, Fact{"seed", seed}) + } + + if outcome.Draws != nil { + facts = append(facts, Fact{"draws", strconv.Itoa(*outcome.Draws)}) + } + + terminal, err := getTerminal(config, run) + if err == nil { + facts = append(facts, Fact{"terminal", describeTerminal(terminal)}) + } + + facts = append(facts, Fact{"timeout", config.Timeout.String()}) + + for _, name := range []string{"LC_ALL", "LANG", "TZ"} { + if value := lookup(environment, name); value != "" { + facts = append(facts, Fact{name, value}) + } + } + + return facts +} + +func describeTerminal(terminal utils.Terminal) string { + if terminal == utils.TerminalPipes { + return "pipes" + } + + columns, rows := utils.TerminalSize() + return fmt.Sprintf("pty, %dx%d", columns, rows) +} + +// childEnv is the environment the program was actually started with, which is +// prescript's own when the run declared none. +func childEnv(run script.Run) []string { + if environment := run.Environment(os.Environ()); environment != nil { + return environment + } + return os.Environ() +} + +func lookup(environment []string, name string) string { + value := "" + // Last assignment wins, the same way exec resolves it. + for _, entry := range environment { + if strings.HasPrefix(entry, name+"=") { + value = strings.TrimPrefix(entry, name+"=") + } + } + return value +} + +// toolchainVersion asks the interpreter what it is. +// +// Only an executable named by name is asked. One named by a path is the +// program under test rather than the toolchain under it, and running somebody +// else's program with an argument it never expected to see is not a thing to +// do behind their back. +func toolchainVersion(executable string) string { + if executable == "" || strings.ContainsRune(executable, filepath.Separator) { + return "" + } + + arguments := []string{"--version"} + if executable == "go" { + arguments = []string{"version"} + } + + // Bounded, because this runs after a failure and must not become a second + // way for the same run to hang. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + output, err := exec.CommandContext(ctx, executable, arguments...).CombinedOutput() + if err != nil && len(output) == 0 { + return "" + } + + for _, line := range strings.Split(string(output), "\n") { + if line = strings.TrimSpace(line); line != "" { + return line + } + } + return "" +} + +// tapeHeader is the provenance the tape carries in its own comments. Quoted +// rather than summarised: which interpreter recorded it, built against which +// generator, from which seed is the difference between a report that can be +// reproduced and one that cannot even in principle. +func tapeHeader(path string) []string { + contents := readFile(path) + if contents == "" { + return nil + } + + var header []string + for _, line := range strings.Split(contents, "\n") { + if !strings.HasPrefix(line, "#") { + break + } + header = append(header, line) + } + return header +} + +func readFile(path string) string { + if path == "" { + return "" + } + + contents, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimRight(string(contents), "\n") +} + +// reproCommand rewrites the command line into one a reader can paste. +// +// argv[0] becomes "prescript", because how this machine happens to reach the +// binary is not part of the bug, and every path inside the working directory +// becomes relative to it -- both so the command works from a checkout and so a +// report can be posted in public without a home directory in it. +func reproCommand(commandLine []string) []string { + if len(commandLine) == 0 { + return nil + } + + working, err := os.Getwd() + if err != nil { + working = "" + } + + command := []string{"prescript"} + for _, argument := range commandLine[1:] { + command = append(command, relativise(argument, working)) + } + return command +} + +func relativise(argument string, working string) string { + if working == "" || !filepath.IsAbs(argument) { + return argument + } + + relative, err := filepath.Rel(working, argument) + if err != nil || strings.HasPrefix(relative, "..") { + return argument + } + return relative +} + +// transcriptLabel is what the expected side is called when there is no +// reference implementation to name -- a single run, or a comparison whose +// reference did not match its own recording. The script is then the only +// authority in the room, and saying so is more honest than naming a port that +// was not consulted. +const transcriptLabel = "the transcript" + +// Markdown is the report as a file. Markdown because it is what the issue +// trackers these end up in accept, and it stays readable in a terminal for +// anyone who just cats it. +// +// The order is the order it gets read in: what went wrong, how to see it +// again, and only then the environment it went wrong in. A reader who already +// knows the bug stops after the first section; a reader who says "works for +// me" is looking for the third. +func (report BugReport) Markdown() string { + var out strings.Builder + + fmt.Fprintf(&out, "# %s\n\n", report.title()) + fmt.Fprintf(&out, "In `%s`, against `%s`.\n", report.Script, report.Reference) + if tokens := report.Finding.Classification.Tokens(); len(tokens) > 0 { + fmt.Fprintf(&out, "Classified `%s`.\n", strings.Join(tokens, ", ")) + } + + out.WriteString("\n## The divergence\n\n") + out.WriteString("```\n") + out.WriteString(report.Finding.report(report.Reference, len(report.Reference))) + out.WriteString("```\n") + + // Only the first. A run that went wrong at step 5 was not really running + // the same program from step 6 onwards, and a report that lists every + // later difference is asking its reader to work out which one is the + // cause. + if later := report.Finding.StepCount - report.Finding.StepIndex - 1; later > 0 && !report.Finding.Matched { + fmt.Fprintf(&out, "\nThis is the first divergence; %s never reached.\n", pluralSteps(later)) + } + + out.WriteString("\n## Reproducing it\n\n") + out.WriteString("```sh\n") + for _, line := range report.repro() { + out.WriteString(line + "\n") + } + out.WriteString("```\n") + + out.WriteString("\n## The environment\n\n") + out.WriteString(factTable(report.Facts)) + + if len(report.TapeHeader) > 0 { + out.WriteString("\nThe tape says of itself:\n\n```\n") + for _, line := range report.TapeHeader { + out.WriteString(line + "\n") + } + out.WriteString("```\n") + } + + if report.Toolchain != "" { + out.WriteString("\nThe image was built with:\n\n```\n") + out.WriteString(report.Toolchain + "\n") + out.WriteString("```\n") + } + + return out.String() +} + +func (report BugReport) title() string { + who := strings.Join(report.Finding.Ports, ", ") + if report.Finding.Matched { + return fmt.Sprintf("%s: %s after all %d steps", who, report.Finding.Mode, report.Finding.StepCount) + } + return fmt.Sprintf("%s: %s at step %d of %d", who, report.Finding.Mode, + report.Finding.StepIndex+1, report.Finding.StepCount) +} + +// repro is the block someone else pastes. Pulling the image comes first +// because it is the step that makes the rest mean anything. +// +// Only a registry digest earns that line. A locally built image is identified +// by its config digest, a bare sha256: with no name in front of it, which +// says which bytes ran but is not something anybody else can pull -- so it is +// named in the table and left out of the commands. A command that names an +// image nobody can fetch is worse than no command. +func (report BugReport) repro() []string { + var lines []string + for _, fact := range report.Facts { + if fact.Name == "image" && strings.Contains(fact.Value, "@sha256:") { + lines = append(lines, "make pull_image PUBLISHED_REF="+fact.Value) + } + } + return append(lines, strings.Join(report.Command, " ")) +} + +// factTable is a two-column table without a header row, because both columns +// are labelled by what is in them and a "name | value" header adds a line +// nobody reads. GitHub needs the separator regardless. +func factTable(facts []Fact) string { + var table strings.Builder + table.WriteString("| | |\n| --- | --- |\n") + for _, fact := range facts { + fmt.Fprintf(&table, "| %s | %s |\n", fact.Name, inlineCode(fact.Value)) + } + return table.String() +} + +// inlineCode quotes a value as code so that a path, a digest or a version +// survives being rendered. A value with a backtick in it is left alone rather +// than escaped: nothing here has one, and a broken table is easier to notice +// than a silently mangled value. +func inlineCode(value string) string { + if value == "" || strings.ContainsRune(value, '`') { + return value + } + return "`" + value + "`" +} + +// WriteBugReport writes the report for these outcomes, if there is one to +// write. It reports whether it wrote anything, and any error is the caller's +// to report: a bug report that could not be written is worth saying out loud, +// because the run it described has already finished and will not be repeated +// for free. +func WriteBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome, commandLine []string) (bool, error) { + report, found := NewBugReport(config, runs, outcomes, commandLine) + if !found { + return false, nil + } + + if err := os.WriteFile(config.BugReport, []byte(report.Markdown()), 0o644); err != nil { + return false, err + } + return true, nil +} diff --git a/internal/play/bug_report_test.go b/internal/play/bug_report_test.go new file mode 100644 index 0000000..b2f8218 --- /dev/null +++ b/internal/play/bug_report_test.go @@ -0,0 +1,260 @@ +package play + +import ( + "os" + "path/filepath" + "strings" + "testing" + + cfg "github.com/rjnienaber/prescript/internal/config" + "github.com/rjnienaber/prescript/internal/script" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func reportConfig() cfg.PlayConfig { + return cfg.PlayConfig{ScriptFile: "dice.yaml", Terminal: "pipes"} +} + +// A report about the second thing that went wrong is a report about a +// consequence. Everything after the first divergence was produced by a program +// that had already stopped running the same script. +func TestOnlyTheFirstDivergenceIsReported(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("python")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("python", atStep(1, "7 807")), + divergedOutcome("ruby", atStep(0, "HOW MANY ROLLS?")), + } + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, []string{"ruby"}, report.Finding.Ports) + assert.Contains(t, report.Markdown(), "# ruby: no-match at step 1 of 2") + assert.NotContains(t, report.Markdown(), "807") +} + +func TestBothSidesAndTheClassificationAreInTheReport(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + } + + markdown := markdownFor(t, reportConfig(), runs, outcomes) + + assert.Contains(t, markdown, "Classified `formatting`") + assert.Contains(t, markdown, " basic \" 7 807 \"") + assert.Contains(t, markdown, " ruby \"7 807\"") +} + +// With one run there is no implementation being treated as correct, and +// naming one that was never played would be a lie about where the expected +// side came from. +func TestASingleRunIsReportedAgainstItsTranscript(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("ruby")} + outcomes := []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))} + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, "the transcript", report.Reference) + assert.Contains(t, report.Markdown(), "against `the transcript`") +} + +// A reference that no longer matches its own recording is the more urgent +// finding, not a reason to write nothing: the environment has moved under a +// script that used to pass. +func TestAReferenceThatDivergedIsItselfTheReport(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + divergedOutcome("basic", atStep(1, "7 806")), + matchedOutcome("ruby"), + } + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, []string{"basic"}, report.Finding.Ports) + assert.Equal(t, "the transcript", report.Reference) +} + +func TestThereIsNothingToReportWhenEveryRunMatched(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{matchedOutcome("basic"), matchedOutcome("ruby")} + + _, found := NewBugReport(reportConfig(), runs, outcomes, nil) + assert.False(t, found) +} + +// The repro line is the command that was actually run, minus the two things +// that are about this machine rather than about the bug: how the binary was +// reached, and where the checkout happens to live. +func TestTheReproCommandIsPasteable(t *testing.T) { + t.Parallel() + + working, err := os.Getwd() + require.NoError(t, err) + + runs := []script.Run{comparableRun("ruby")} + outcomes := []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))} + commandLine := []string{"/Users/someone/go/bin/prescript", "play", + filepath.Join(working, "dice.yaml"), "--tape", "/elsewhere/dice.tape"} + + report, found := NewBugReport(reportConfig(), runs, outcomes, commandLine) + require.True(t, found) + + assert.Equal(t, + []string{"prescript", "play", "dice.yaml", "--tape", "/elsewhere/dice.tape"}, + report.Command) + assert.Contains(t, report.Markdown(), "prescript play dice.yaml --tape /elsewhere/dice.tape") +} + +func TestAnUnpinnedRunSaysSoRatherThanLeavingAGap(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| image | `none; this did not run in the pinned environment` |") + assert.NotContains(t, markdown, "make pull_image") +} + +// A digest is the only thing that makes the rest of the report reproducible, +// so when there is one it becomes the first line of the repro block. +func TestAPinnedRunPullsTheImageFirst(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "ghcr.io/rjnienaber/prescript-env@sha256:abc123") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, + "make pull_image PUBLISHED_REF=ghcr.io/rjnienaber/prescript-env@sha256:abc123") +} + +// A locally built image has a config digest and no name anybody else can pull +// it by. It says which bytes ran, which is worth recording, and it is not a +// command anyone can paste. +func TestALocallyBuiltImageIsNamedButNotPulled(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "sha256:9f8e7d") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| image | `sha256:9f8e7d` |") + assert.NotContains(t, markdown, "make pull_image") +} + +// Without the tape the finding is not reproducible even in principle, so what +// the tape says about where it came from is quoted rather than summarised. +func TestTheTapeAndItsProvenanceAreQuoted(t *testing.T) { + t.Parallel() + + tape := filepath.Join(t.TempDir(), "dice.tape") + require.NoError(t, os.WriteFile(tape, + []byte("# recorded from vintbas 0.4.0, seed 0\n# 3 values\n0.5\n0.25\n0.75\n"), 0o600)) + + config := reportConfig() + config.Tape = tape + + markdown := markdownFor(t, config, []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| tape | `"+tape+"` |") + assert.Contains(t, markdown, "# recorded from vintbas 0.4.0, seed 0") + assert.Contains(t, markdown, "# 3 values") + assert.NotContains(t, markdown, "0.25") +} + +// Without a tape the seed is what makes the run repeatable, and it belongs to +// the run rather than to prescript. +func TestTheSeedStandsInForTheTape(t *testing.T) { + t.Parallel() + + run := comparableRun("ruby") + run.Env = map[string]string{"PRESCRIPT_SEED": "42", "TZ": "UTC"} + + markdown := markdownFor(t, reportConfig(), []script.Run{run}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| seed | `42` |") + assert.Contains(t, markdown, "| TZ | `UTC` |") +} + +func TestTheDrawCountIsReportedOnlyWhenItIsKnown(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("ruby")} + outcome := divergedOutcome("ruby", atStep(1, "7 807")) + + assert.NotContains(t, markdownFor(t, reportConfig(), runs, []Outcome{outcome}), "| draws |") + + outcome.Draws = drew(12) + assert.Contains(t, markdownFor(t, reportConfig(), runs, []Outcome{outcome}), "| draws | `12` |") +} + +func TestTheTerminalAndPrescriptsOwnVersionAreReported(t *testing.T) { + t.Parallel() + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| terminal | `pipes` |") + assert.Contains(t, markdown, "| prescript |") + assert.Contains(t, markdown, "| timeout |") +} + +// A program named by a path is the thing under test. Running somebody else's +// program with an argument it never expected is not a thing to do behind their +// back, and the answer would be the port's version rather than the toolchain's +// anyway. +func TestOnlyAnExecutableOnThePathIsAskedForItsVersion(t *testing.T) { + t.Parallel() + + assert.Empty(t, toolchainVersion("/usr/bin/ruby")) + assert.Empty(t, toolchainVersion("./dice")) + assert.Empty(t, toolchainVersion("")) + assert.Empty(t, toolchainVersion("prescript-no-such-executable")) +} + +func TestTheReportIsWrittenToTheNamedFile(t *testing.T) { + t.Parallel() + + config := reportConfig() + config.BugReport = filepath.Join(t.TempDir(), "report.md") + + written, err := WriteBugReport(config, []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}, nil) + require.NoError(t, err) + assert.True(t, written) + + contents, err := os.ReadFile(config.BugReport) + require.NoError(t, err) + assert.Contains(t, string(contents), "# ruby: no-match at step 2 of 2") +} + +func markdownFor(t *testing.T, config cfg.PlayConfig, runs []script.Run, outcomes []Outcome) string { + t.Helper() + + report, found := NewBugReport(config, runs, outcomes, nil) + require.True(t, found) + + markdown := report.Markdown() + // Every section is required: a report missing one of them is the kind of + // report that gets answered with "works for me". + for _, heading := range []string{"## The divergence", "## Reproducing it", "## The environment"} { + assert.True(t, strings.Contains(markdown, heading), heading) + } + return markdown +} diff --git a/internal/play/runner_integration_test.go b/internal/play/runner_integration_test.go index 5903bda..a326a33 100644 --- a/internal/play/runner_integration_test.go +++ b/internal/play/runner_integration_test.go @@ -337,3 +337,40 @@ func TestTheDrawCountSurvivesARunThatIsKilled(t *testing.T) { }) } } + +// The report end to end, against a language that is actually installed: the +// facts a unit test has to stub -- what the interpreter says it is, how much +// of the tape the run drew, what the tape says about where it came from -- are +// exactly the ones that make a report answerable, so at least once they are +// gathered from a real run rather than described. +func TestABugReportGathersTheFactsFromARealRun(t *testing.T) { + t.Parallel() + runner := loadRunner(t, "ruby.yaml") + + // The reference's own draws, minus one, so the run diverges on the last + // line having drawn the whole tape's worth first. + wrong := strings.Replace(referenceDraws, "973", "972", 1) + runs := script.ApplyRunner([]script.Run{drawingRun(t, "rng.rb", wrong)}, runner) + + config := tapedConfig(t) + config.ScriptFile = "rng.yaml" + config.BugReport = filepath.Join(t.TempDir(), "report.md") + + outcome := Run(config, runs[0], &utils.CustomLogger{}) + assert.NotEqual(t, 0, outcome.ExitCode) + + written, err := WriteBugReport(config, runs, []Outcome{outcome}, + []string{"/somewhere/prescript", "play", "rng.yaml", "--tape", tapePath(t)}) + assert.NoError(t, err) + assert.True(t, written) + + contents, err := os.ReadFile(config.BugReport) + assert.NoError(t, err) + report := string(contents) + + assert.Contains(t, report, "| ruby | `ruby ") + assert.Contains(t, report, "| draws | `5` |") + assert.Contains(t, report, "# source: vintage-basic") + assert.Contains(t, report, "prescript play rng.yaml --tape "+tapePath(t)) + assert.Contains(t, report, referenceDraws) +} diff --git a/internal/play/runs.go b/internal/play/runs.go index 8108d3a..fd60f81 100644 --- a/internal/play/runs.go +++ b/internal/play/runs.go @@ -50,9 +50,33 @@ func RunAll(config cfg.PlayConfig, runs []script.Run, logger utils.Logger) int { fmt.Fprint(os.Stderr, Compare(runs, outcomes).Report()) } + if config.BugReport != "" { + writeBugReport(config, runs, outcomes) + } + return result } +// writeBugReport is best-effort and says so either way. It runs after every +// run has finished, so a failure here cannot be retried without paying for the +// whole script again -- which is exactly why it is reported rather than +// swallowed, and why it does not change the exit code: the runs already +// decided that, and a bug report that could not be filed is not a run that +// passed or failed differently. +func writeBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome) { + // os.Args rather than the parsed config, because the point of the repro + // line is to be the command that was actually run. + written, err := WriteBugReport(config, runs, outcomes, os.Args) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "could not write bug report %s: %s\n", config.BugReport, err) + case written: + fmt.Fprintf(os.Stderr, "\nbug report written to %s\n", config.BugReport) + default: + fmt.Fprintf(os.Stderr, "\nnothing to report: no run diverged\n") + } +} + // warnAboutOverrides is a warning rather than a refusal. Overriding one thing // for every run is meaningful when the runs differ in their steps, and a // mistake when they differ in their executable, and prescript cannot tell which diff --git a/internal/utils/pty_unix.go b/internal/utils/pty_unix.go index 787b98b..acaadfd 100644 --- a/internal/utils/pty_unix.go +++ b/internal/utils/pty_unix.go @@ -11,20 +11,6 @@ import ( "golang.org/x/term" ) -// The size every one of these programs was written for, and the size the pty -// is given whatever terminal prescript itself was started from. -// -// A pty opened without a size starts at zero rows and columns, so a program -// that asks how wide its terminal is -- to decide how many columns of a table -// to print, or where to wrap -- gets an answer that is neither what a real -// terminal would give it nor what the next machine's terminal would. Either -// way that is a difference in output with no difference in behaviour behind -// it, which is exactly the kind of noise a comparison run has to be free of. -const ( - terminalColumns = 80 - terminalRows = 24 -) - // startPty starts cmd with a pseudo-terminal on all three of its standard // streams and returns this process's end of it. // diff --git a/internal/utils/terminal_size.go b/internal/utils/terminal_size.go new file mode 100644 index 0000000..34f02c5 --- /dev/null +++ b/internal/utils/terminal_size.go @@ -0,0 +1,24 @@ +package utils + +// The size every one of these programs was written for, and the size the pty +// is given whatever terminal prescript itself was started from. +// +// A pty opened without a size starts at zero rows and columns, so a program +// that asks how wide its terminal is -- to decide how many columns of a table +// to print, or where to wrap -- gets an answer that is neither what a real +// terminal would give it nor what the next machine's terminal would. Either +// way that is a difference in output with no difference in behaviour behind +// it, which is exactly the kind of noise a comparison run has to be free of. +const ( + terminalColumns = 80 + terminalRows = 24 +) + +// TerminalSize is the size prescript gives every child's pty, in columns and +// rows. Exported because a bug report has to name it: a program that wraps its +// output at the terminal's width prints something different at a different +// width, and a reader who cannot see which width was used cannot tell that +// from a real difference. +func TerminalSize() (int, int) { + return terminalColumns, terminalRows +} diff --git a/internal/utils/version.go b/internal/utils/version.go new file mode 100644 index 0000000..595c1c7 --- /dev/null +++ b/internal/utils/version.go @@ -0,0 +1,57 @@ +package utils + +import ( + "runtime/debug" +) + +// version can be stamped in at build time with +// -ldflags "-X github.com/rjnienaber/prescript/internal/utils.version=1.2.3". +// Left empty by an ordinary build, which falls back to what the toolchain +// records for itself. +var version = "" + +// Version identifies the prescript that produced a report. +// +// A bug report that does not say which build made it cannot be answered: the +// first question about any cross-implementation report is whether the tool +// still behaves that way, and the second is which tool. The Go toolchain +// stamps the commit and whether the tree was dirty into every binary built +// from a repository, so this is available without a release process to +// maintain -- and a build made from uncommitted changes says so, which is the +// case most worth knowing about. +func Version() string { + if version != "" { + return version + } + + info, available := debug.ReadBuildInfo() + if !available { + return "unknown" + } + + revision := "" + modified := false + for _, setting := range info.Settings { + switch setting.Key { + case "vcs.revision": + revision = setting.Value + case "vcs.modified": + modified = setting.Value == "true" + } + } + + if revision == "" { + if info.Main.Version != "" { + return info.Main.Version + } + return "unknown" + } + + if len(revision) > 12 { + revision = revision[:12] + } + if modified { + return revision + " (modified)" + } + return revision +} From 0512c13d6d31472b97211c2df3d851e19081b045 Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 22:57:49 +0100 Subject: [PATCH 2/4] let git read the mounted repository inside the container The repository is mounted from the host, so its files belong to a user the container does not have, and git refuses to read a repository it considers somebody else's. That was harmless until the binary started carrying its own revision: `go build` asks git for it and treats the refusal as an error rather than as "no VCS here", so every build inside the image failed with `error obtaining VCS status: exit status 128`. Trusted for the one directory that is ever mounted, rather than turning VCS stamping off -- the version in a bug report is the whole point, and a report written inside the pinned environment is the one most likely to be filed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- docker/Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f59363b..3eb9e04 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,7 +89,8 @@ ENV LANG=C.UTF-8 \ # What a bug report has to carry, written from the binaries that were actually # installed rather than from the arguments above, so it cannot claim a version -# the image does not have. Issue #18 reads this file; a human can cat it. +# the image does not have. `prescript play --bug-report` quotes it verbatim; a +# human can cat it. ENV PRESCRIPT_ENVIRONMENT=/etc/prescript-environment RUN { \ echo "base=debian:bookworm-slim@${BASE_DIGEST}"; \ @@ -106,6 +107,14 @@ RUN { \ COPY entrypoint.sh /usr/local/bin/prescript-entrypoint RUN chmod +x /usr/local/bin/prescript-entrypoint +# The repository is mounted from the host, so its files belong to a user this +# container does not have, and git refuses to read a repository it considers +# somebody else's. `go build` asks git for the revision it stamps into the +# binary and treats that refusal as an error rather than as "no VCS here", so +# without this every build inside the image fails. Scoped to the one directory +# that is ever mounted. +RUN git config --system --add safe.directory /workspace + WORKDIR /workspace ENTRYPOINT ["/usr/local/bin/prescript-entrypoint"] CMD ["bash"] From 0dff462d78e684576d8a35ccace73a6207484c9a Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 22:59:59 +0100 Subject: [PATCH 3/4] assert on the finding rather than on a string that can collide `should not contain "807"` was meant to say the later finding is not in the report. Inside the container it also matched the image's own digest, which is hex and long enough to contain any three digits sooner or later. Says what it meant instead: the port that diverged later is not named, and neither is its step. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- internal/play/bug_report_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/play/bug_report_test.go b/internal/play/bug_report_test.go index b2f8218..1e13345 100644 --- a/internal/play/bug_report_test.go +++ b/internal/play/bug_report_test.go @@ -34,7 +34,8 @@ func TestOnlyTheFirstDivergenceIsReported(t *testing.T) { assert.Equal(t, []string{"ruby"}, report.Finding.Ports) assert.Contains(t, report.Markdown(), "# ruby: no-match at step 1 of 2") - assert.NotContains(t, report.Markdown(), "807") + assert.NotContains(t, report.Markdown(), "python") + assert.NotContains(t, report.Markdown(), "step 2 of 2") } func TestBothSidesAndTheClassificationAreInTheReport(t *testing.T) { From ee8a795a7ad5636ed0371e03746974f636fe9cc5 Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 23:02:02 +0100 Subject: [PATCH 4/4] scope the first-divergence assertion to the section it is about Twice now a NotContains over the whole report has matched something the machine put there rather than something the report got wrong: the image digest is hex long enough to contain any three digits, and the image manifest names every language installed, python included. Asserted against the divergence section alone, which is the part the test is about. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- internal/play/bug_report_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/play/bug_report_test.go b/internal/play/bug_report_test.go index 1e13345..dad6107 100644 --- a/internal/play/bug_report_test.go +++ b/internal/play/bug_report_test.go @@ -34,8 +34,28 @@ func TestOnlyTheFirstDivergenceIsReported(t *testing.T) { assert.Equal(t, []string{"ruby"}, report.Finding.Ports) assert.Contains(t, report.Markdown(), "# ruby: no-match at step 1 of 2") - assert.NotContains(t, report.Markdown(), "python") - assert.NotContains(t, report.Markdown(), "step 2 of 2") + + divergence := section(t, report.Markdown(), "## The divergence") + assert.Contains(t, divergence, "step 1 of 2") + assert.NotContains(t, divergence, "python") + assert.NotContains(t, divergence, "807") +} + +// section is the body of one heading. Scoped, because the rest of the report +// quotes whatever the machine happens to be -- an image digest is hex long +// enough to contain any three digits, and the image manifest names every +// language installed -- so a NotContains over the whole file asserts something +// about this machine rather than about the report. +func section(t *testing.T, markdown string, heading string) string { + t.Helper() + + _, body, found := strings.Cut(markdown, heading) + require.True(t, found, heading) + + if next := strings.Index(body, "\n## "); next >= 0 { + return body[:next] + } + return body } func TestBothSidesAndTheClassificationAreInTheReport(t *testing.T) {