diff --git a/Makefile b/Makefile index f605d35..2c7ad11 100644 --- a/Makefile +++ b/Makefile @@ -94,11 +94,11 @@ lint: .PHONY: build_dev build_dev: - go build -o tmp/prescript cmd/prescript/main.go + go build -o tmp/prescript ./cmd/prescript .PHONY: build_release build_release: - go build -ldflags="-s -w" -a -o tmp/prescript cmd/prescript/main.go + go build -ldflags="-s -w" -a -o tmp/prescript ./cmd/prescript .PHONY: test test: diff --git a/README.md b/README.md index e0aaa32..61061ca 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ prescript play [script file] [flags] -- [executable arguments] | `-t` | timeout waiting for output from external command | `bool` | `30s ` | No | | `--terminal` | what the executable is given for its standard streams (`pty` or `pipes`) | `enum` | `pty` | No | | `--tape` | file of recorded random values to replay to the executable | `string` | | No | +| `--bug-report` | write the first divergence to this file as a report someone else can act on | `string` | | No | | `-- [args]` | arguments for the executable, overriding those in the script | `list` | | No | #### `record` @@ -298,6 +299,24 @@ one report per language rather than one per program. `no-draws` means the port had taken nothing from the tape, so randomness is not involved. See [docs/comparison.md](docs/comparison.md). +### Filing what it found + +The usual answer to a cross-implementation bug report is "works for me", and it +is usually fair: the two people ran different interpreter builds, in different +locales, against different random numbers. `--bug-report` writes the first +divergence to a file with all of that pinned down, because the run already knew +it: + +```bash +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +The file holds both sides of the divergence and its classification, the +interpreter version, the container digest, the tape and how much of it the port +drew, the locale, timezone and terminal size, prescript's own revision, and one +command that reproduces the lot. It files nothing and changes no exit code. See +[docs/bug-reports.md](docs/bug-reports.md). + ### Development ``` diff --git a/cmd/prescript/main.go b/cmd/prescript/main.go index 3b55bf2..2b7fd09 100644 --- a/cmd/prescript/main.go +++ b/cmd/prescript/main.go @@ -14,7 +14,15 @@ import ( func main() { config, err := cfg.GetConfig() - if err != nil || config.Subcommand == cfg.NoCommand { + if err != nil { + os.Exit(utils.USER_ERROR) + } + + if config.Handled { + os.Exit(utils.SUCCESS) + } + + if config.Subcommand == cfg.NoCommand { os.Exit(utils.USER_ERROR) } diff --git a/docker/Dockerfile b/docker/Dockerfile index f59363b..3eb9e04 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,7 +89,8 @@ ENV LANG=C.UTF-8 \ # What a bug report has to carry, written from the binaries that were actually # installed rather than from the arguments above, so it cannot claim a version -# the image does not have. Issue #18 reads this file; a human can cat it. +# the image does not have. `prescript play --bug-report` quotes it verbatim; a +# human can cat it. ENV PRESCRIPT_ENVIRONMENT=/etc/prescript-environment RUN { \ echo "base=debian:bookworm-slim@${BASE_DIGEST}"; \ @@ -106,6 +107,14 @@ RUN { \ COPY entrypoint.sh /usr/local/bin/prescript-entrypoint RUN chmod +x /usr/local/bin/prescript-entrypoint +# The repository is mounted from the host, so its files belong to a user this +# container does not have, and git refuses to read a repository it considers +# somebody else's. `go build` asks git for the revision it stamps into the +# binary and treats that refusal as an error rather than as "no VCS here", so +# without this every build inside the image fails. Scoped to the one directory +# that is ever mounted. +RUN git config --system --add safe.directory /workspace + WORKDIR /workspace ENTRYPOINT ["/usr/local/bin/prescript-entrypoint"] CMD ["bash"] diff --git a/docs/bug-reports.md b/docs/bug-reports.md new file mode 100644 index 0000000..4e0272f --- /dev/null +++ b/docs/bug-reports.md @@ -0,0 +1,179 @@ +# Bug reports + +``` +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +`--bug-report` writes the first divergence to a file, as Markdown, with +everything somebody else needs to reproduce it. + +## Why prescript writes it rather than you + +The default answer to a cross-implementation bug report is "works for me", and +it is usually a fair answer. Two people running the same program in two +languages are also running two interpreter builds, two locales, and two +sequences of random numbers, and any one of those explains a difference on its +own. A report that does not pin all of them down cannot be acted on, only +argued with. + +Pinning them down by hand is perhaps ten minutes of copying version strings out +of terminals. Nobody does it twice, so the second report is worse than the +first, and by the tenth the reports have stopped being filed at all. Everything +in the block below is something the run already knew about itself, so it is +written down rather than remembered. + +## What it writes + +````markdown +# ruby: no-match at step 5 of 12 + +In `dice.yaml`, against `basic`. +Classified `formatting, same-draws`. + +## The divergence + +``` +step 5 of 12 (no-match, formatting, same-draws) + basic " 7 807 " + ruby "7 807" + ^ first difference +``` + +This is the first divergence; 7 later steps were never reached. + +## Reproducing it + +```sh +make pull_image PUBLISHED_REF=ghcr.io/rjnienaber/prescript-env@sha256:2a1f... +prescript play dice.yaml --tape tapes/vintbas-seed0.tape --bug-report dice.md +``` + +## The environment + +| | | +| --- | --- | +| prescript | `86fa312b39df` | +| script | `dice.yaml` | +| command | `ruby ports/ruby/dice.rb` | +| ruby | `ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-linux]` | +| image | `ghcr.io/rjnienaber/prescript-env@sha256:2a1f...` | +| tape | `tapes/vintbas-seed0.tape` | +| draws | `5` | +| terminal | `pty, 80x24` | +| timeout | `30s` | +| LC_ALL | `C.UTF-8` | +| TZ | `UTC` | + +The tape says of itself: + +``` +# prescript tape 1 +# source: vintage-basic 1.0.3, built as vintbas-1.0.3-1 +# random: 1.2.1.2 +# seed: 0 +# values: 10000 +``` +```` + +The three sections are in the order they get read in. Somebody who already +knows the bug stops after the first; somebody about to say "works for me" is +looking for the third. + +## The first divergence only + +A run that went wrong at step 5 was not really running the same program from +step 6 onwards. Everything after the first difference is a consequence of it, +and a report that lists all of them is asking its reader to work out which one +was the cause — which is the work the report was supposed to do. + +Which divergence that is depends on what was played: + +- **A script with several runs.** The earliest finding in the + [comparison](comparison.md): the first step at which any port left the + reference, and every port that left it there in the same way. Findings are + grouped, so a report about one bug in twelve ports is one report naming + twelve ports. +- **A script whose reference failed.** The reference's own divergence. That is + the more urgent finding rather than a reason to write nothing: the + environment has moved under a script that used to pass, and the ports were + never compared against anything. +- **A single run.** Its own failure, reported against `the transcript`. There is + no implementation being treated as correct, and naming one that was never + played would misdescribe where the expected side came from. + +When every run matched there is nothing to write, and prescript says so instead +of writing an empty file. + +## What is in the facts table, and why + +Each row is there because it has been the answer to a "works for me" at least +once. + +| Row | Why it is there | +| --- | --- | +| `prescript` | The revision that produced the report, from the binary's own build info. Reports outlive releases. | +| `script` | Which script, since one corpus has hundreds. | +| `command` | The executable and arguments the run actually used, after any runner and any `--exec`. | +| the interpreter | What the toolchain says it is. See below. | +| `image` | The pinned digest, which is what makes everything else reproducible. | +| `tape` or `seed` | Where the randomness came from. Without it the report is not reproducible even in principle. | +| `draws` | How much of the tape the port had used when it stopped. Zero means randomness is not implicated at all. | +| `terminal` | `pty, 80x24` or `pipes`. Programs lay their output out differently depending on the answer, and a size is not something a reader can guess. | +| `timeout` | What a timeout meant on this run. | +| `LC_ALL`, `LANG`, `TZ` | The three variables most likely to change a program's output between two machines that are otherwise identical. | + +The locale and timezone are read from the environment the program was actually +started with, which is the run's own when it +[declared one](script-format.md) and prescript's otherwise. + +### Asking the interpreter what it is + +The executable is run once more with `--version` (`version` for `go`), and the +first line of its output becomes a row. Only an executable named by name is +asked. One named by a path — `./dice`, `ports/ruby/dice.rb` — is the program +under test rather than the toolchain under it, and running somebody else's +program with an argument it never expected is not a thing to do behind their +back. The probe is bounded at five seconds and omitted entirely if it fails, so +it cannot become a second way for a failed run to hang. + +### The image + +A run outside the [pinned container](container.md) says so, in the table, in +those words: + +``` +| image | `none; this did not run in the pinned environment` | +``` + +That is a fact worth stating rather than a gap worth leaving. It is the single +likeliest reason the reader cannot reproduce what they are being shown. + +The `make pull_image` line appears in the repro block only when the digest is +one somebody else can pull. A locally built image is identified by its config +digest — a bare `sha256:` with no name in front of it — which says which bytes +ran but cannot be fetched, so it is named in the table and left out of the +commands. + +## The repro command + +The command line is reproduced as it was given, with two changes: + +- `argv[0]` becomes `prescript`. How this machine reaches the binary is not + part of the bug. +- Absolute paths inside the working directory become relative to it. This makes + the command work from a fresh checkout, and it keeps a home directory out of + a report that is about to be posted in public. + +Paths outside the working directory are left alone: shortening them would +produce a command that does not work. If a report is going somewhere public, +this is the line to read before pasting it. + +## What it does not do + +It does not file anything. The output is a file; where it goes is a decision +with a person in it. + +It does not change the exit code. Whether the runs passed was decided by the +runs. A report that could not be written is said out loud on stderr — the run +it described has already finished, and will not be repeated for free — but it +is not itself a failure of the run. diff --git a/docs/comparison.md b/docs/comparison.md index 01ce449..bc2acd5 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -186,3 +186,9 @@ different output belong in two scripts. `prescript` still exits with the first non-zero code any run produced. The comparison is a report, not a verdict: it says where the implementations parted company, and what to do about that is the reader's call. + +## Filing a finding + +`--bug-report ` writes the earliest finding out as Markdown, with the +interpreter version, the container digest, the tape and everything else needed +to answer "works for me". See [bug-reports.md](bug-reports.md). diff --git a/internal/config/play.go b/internal/config/play.go index 39d6641..948ba49 100644 --- a/internal/config/play.go +++ b/internal/config/play.go @@ -16,6 +16,7 @@ type PlayConfig struct { RunnerFile string Terminal string Tape string + BugReport string Arguments []string } @@ -39,6 +40,7 @@ func createPlaySubCommand(config *Config) *cobra.Command { playCmd.Flags().StringVarP(&config.Play.RunnerFile, "runner", "r", "", "runner file describing how to launch an implementation") playCmd.Flags().StringVar(&config.Play.Terminal, "terminal", "", "override what the executable is given for its standard streams (pty or pipes)") playCmd.Flags().StringVar(&config.Play.Tape, "tape", "", "file of recorded random values to replay to the executable, instead of letting it draw its own") + playCmd.Flags().StringVar(&config.Play.BugReport, "bug-report", "", "write the first divergence to this file as a report someone else can act on") defaultTimeout := 30 * time.Second playCmd.Flags().DurationVarP(&config.Play.Timeout, "timeout", "t", defaultTimeout, "timeout waiting for output from external command") diff --git a/internal/config/root.go b/internal/config/root.go index 23ef516..8113346 100644 --- a/internal/config/root.go +++ b/internal/config/root.go @@ -14,6 +14,12 @@ const ( ) type Config struct { + // Handled is set when cobra has already answered the command line itself + // -- `--version` prints and there is nothing left to run. It is not a + // failure, and a caller that treats "no subcommand" as a usage error would + // otherwise exit non-zero having done exactly what was asked. + Handled bool + Subcommand subcommand Play PlayConfig Record RecordConfig @@ -30,6 +36,9 @@ func GetConfig() (Config, error) { Use: "prescript [cmd] [script file] [optional executable]", Short: "Record and playback responses to an interactive cli", Long: "Record and playback responses to an interactive cli", + // The same string a generated bug report names itself by, so someone + // reading one can check what they have against what made it. + Version: utils.Version(), } rootCmd.AddCommand(playCmd) @@ -40,5 +49,9 @@ func GetConfig() (Config, error) { return Config{}, err } + if asked, _ := rootCmd.Flags().GetBool("version"); asked { + config.Handled = true + } + return config, nil } diff --git a/internal/play/bug_report.go b/internal/play/bug_report.go new file mode 100644 index 0000000..cf1fc7d --- /dev/null +++ b/internal/play/bug_report.go @@ -0,0 +1,446 @@ +package play + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + cfg "github.com/rjnienaber/prescript/internal/config" + "github.com/rjnienaber/prescript/internal/script" + "github.com/rjnienaber/prescript/internal/utils" +) + +// BugReport is one divergence written up so that somebody else can act on it +// without re-deriving the setup. +// +// The default answer to any cross-implementation bug report is "works for me", +// and it is usually a fair answer: the two people ran different interpreter +// builds, in different locales, against different random numbers. Only a +// complete repro block prevents it, and assembling one by hand for every +// finding is work nobody does twice. So it is assembled here, from what the +// run already knows about itself. +// +// The first divergence only. A run that went wrong at step 5 was not really +// running the program from step 6 onwards, and everything after the first +// difference is a consequence of it. +type BugReport struct { + Script string + Reference string + Finding DivergenceGroup + + // Command reproduces the run. It is the command line prescript was given, + // which is the honest answer to "what did you run" -- rewritten only to + // drop the path prescript itself was invoked by and to make repository + // paths relative, so a report can be pasted somewhere public without + // carrying a home directory into it. + Command []string + + // Toolchain is what the pinned environment says about itself: the base + // image digest and the version of every interpreter in it, written at + // build time from the binaries actually installed. Empty outside it. + Toolchain string + + // TapeHeader is the provenance the tape carries: which interpreter + // recorded it, against which generator, from which seed. + TapeHeader []string + + Facts []Fact +} + +// Fact is one row of the environment table. Ordered, because the order is the +// argument: what was run, then what it was run against, then what was around +// it. +type Fact struct { + Name string + Value string +} + +// NewBugReport picks the divergence worth reporting and gathers everything +// around it. It reports false when there is nothing to report -- every run +// passed, or the runs that failed never got as far as diverging. +func NewBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome, commandLine []string) (BugReport, bool) { + reference, finding, found := firstFinding(runs, outcomes) + if !found { + return BugReport{}, false + } + + run, outcome := runFor(runs, outcomes, finding.Ports[0]) + report := BugReport{ + Script: config.ScriptFile, + Reference: reference, + Finding: finding, + Command: reproCommand(commandLine), + Toolchain: readFile(os.Getenv("PRESCRIPT_ENVIRONMENT")), + TapeHeader: tapeHeader(config.Tape), + } + report.Facts = gatherFacts(config, run, outcome) + return report, true +} + +// firstFinding is the divergence to report: the earliest one in a comparison, +// or the single run's own when there is nothing to compare it against. +func firstFinding(runs []script.Run, outcomes []Outcome) (string, DivergenceGroup, bool) { + if len(outcomes) > 1 { + comparison := Compare(runs, outcomes) + if len(comparison.Groups) > 0 { + return comparison.Reference, comparison.Groups[0], true + } + + // A reference that does not match its own transcript is still a + // finding, and often the more urgent one: the environment has moved + // under a script that used to pass. + if comparison.ReferenceFailure != nil { + return transcriptLabel, asGroup(comparison.Reference, *comparison.ReferenceFailure), true + } + return "", DivergenceGroup{}, false + } + + if len(outcomes) == 1 && outcomes[0].Failure != nil { + return transcriptLabel, asGroup(outcomes[0].Name, *outcomes[0].Failure), true + } + return "", DivergenceGroup{}, false +} + +// asGroup makes one run's divergence look like a finding of one, so a report +// about a single run and a report about a comparison are the same report. +// There is no reference implementation on this path, so the expected line +// speaks for itself and is labelled as what the script asked for. +func asGroup(name string, divergence Divergence) DivergenceGroup { + group := DivergenceGroup{ + StepIndex: divergence.StepIndex, + StepCount: divergence.StepCount, + Mode: divergence.Mode, + Expected: divergence.Expected, + Redacted: divergence.Redacted, + Received: divergence.Lines(), + Matched: divergence.Matched(), + Ports: []string{name}, + } + if group.Matched { + group.Received = []string{divergence.Detail} + } + group.Classification = group.classify(nil, []*int{nil}) + return group +} + +func runFor(runs []script.Run, outcomes []Outcome, name string) (script.Run, Outcome) { + for index, outcome := range outcomes { + if outcome.Name == name { + return runs[index], outcome + } + } + return script.Run{}, Outcome{} +} + +func gatherFacts(config cfg.PlayConfig, run script.Run, outcome Outcome) []Fact { + facts := []Fact{ + {"prescript", utils.Version()}, + {"script", config.ScriptFile}, + {"command", strings.TrimSpace(run.Executable + " " + strings.Join(run.Arguments, " "))}, + } + + if toolchain := toolchainVersion(run.Executable); toolchain != "" { + facts = append(facts, Fact{run.Executable, toolchain}) + } + + // An unpinned run is worth saying out loud rather than leaving as a gap. + // It is the single likeliest reason a reader cannot reproduce what they + // are being shown. + if image := os.Getenv("PRESCRIPT_IMAGE"); image != "" { + facts = append(facts, Fact{"image", image}) + } else { + facts = append(facts, Fact{"image", "none; this did not run in the pinned environment"}) + } + + environment := childEnv(run) + if config.Tape != "" { + facts = append(facts, Fact{"tape", config.Tape}) + } else if seed := lookup(environment, "PRESCRIPT_SEED"); seed != "" { + facts = append(facts, Fact{"seed", seed}) + } + + if outcome.Draws != nil { + facts = append(facts, Fact{"draws", strconv.Itoa(*outcome.Draws)}) + } + + terminal, err := getTerminal(config, run) + if err == nil { + facts = append(facts, Fact{"terminal", describeTerminal(terminal)}) + } + + facts = append(facts, Fact{"timeout", config.Timeout.String()}) + + for _, name := range []string{"LC_ALL", "LANG", "TZ"} { + if value := lookup(environment, name); value != "" { + facts = append(facts, Fact{name, value}) + } + } + + return facts +} + +func describeTerminal(terminal utils.Terminal) string { + if terminal == utils.TerminalPipes { + return "pipes" + } + + columns, rows := utils.TerminalSize() + return fmt.Sprintf("pty, %dx%d", columns, rows) +} + +// childEnv is the environment the program was actually started with, which is +// prescript's own when the run declared none. +func childEnv(run script.Run) []string { + if environment := run.Environment(os.Environ()); environment != nil { + return environment + } + return os.Environ() +} + +func lookup(environment []string, name string) string { + value := "" + // Last assignment wins, the same way exec resolves it. + for _, entry := range environment { + if strings.HasPrefix(entry, name+"=") { + value = strings.TrimPrefix(entry, name+"=") + } + } + return value +} + +// toolchainVersion asks the interpreter what it is. +// +// Only an executable named by name is asked. One named by a path is the +// program under test rather than the toolchain under it, and running somebody +// else's program with an argument it never expected to see is not a thing to +// do behind their back. +func toolchainVersion(executable string) string { + if executable == "" || strings.ContainsRune(executable, filepath.Separator) { + return "" + } + + arguments := []string{"--version"} + if executable == "go" { + arguments = []string{"version"} + } + + // Bounded, because this runs after a failure and must not become a second + // way for the same run to hang. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + output, err := exec.CommandContext(ctx, executable, arguments...).CombinedOutput() + if err != nil && len(output) == 0 { + return "" + } + + for _, line := range strings.Split(string(output), "\n") { + if line = strings.TrimSpace(line); line != "" { + return line + } + } + return "" +} + +// tapeHeader is the provenance the tape carries in its own comments. Quoted +// rather than summarised: which interpreter recorded it, built against which +// generator, from which seed is the difference between a report that can be +// reproduced and one that cannot even in principle. +func tapeHeader(path string) []string { + contents := readFile(path) + if contents == "" { + return nil + } + + var header []string + for _, line := range strings.Split(contents, "\n") { + if !strings.HasPrefix(line, "#") { + break + } + header = append(header, line) + } + return header +} + +func readFile(path string) string { + if path == "" { + return "" + } + + contents, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimRight(string(contents), "\n") +} + +// reproCommand rewrites the command line into one a reader can paste. +// +// argv[0] becomes "prescript", because how this machine happens to reach the +// binary is not part of the bug, and every path inside the working directory +// becomes relative to it -- both so the command works from a checkout and so a +// report can be posted in public without a home directory in it. +func reproCommand(commandLine []string) []string { + if len(commandLine) == 0 { + return nil + } + + working, err := os.Getwd() + if err != nil { + working = "" + } + + command := []string{"prescript"} + for _, argument := range commandLine[1:] { + command = append(command, relativise(argument, working)) + } + return command +} + +func relativise(argument string, working string) string { + if working == "" || !filepath.IsAbs(argument) { + return argument + } + + relative, err := filepath.Rel(working, argument) + if err != nil || strings.HasPrefix(relative, "..") { + return argument + } + return relative +} + +// transcriptLabel is what the expected side is called when there is no +// reference implementation to name -- a single run, or a comparison whose +// reference did not match its own recording. The script is then the only +// authority in the room, and saying so is more honest than naming a port that +// was not consulted. +const transcriptLabel = "the transcript" + +// Markdown is the report as a file. Markdown because it is what the issue +// trackers these end up in accept, and it stays readable in a terminal for +// anyone who just cats it. +// +// The order is the order it gets read in: what went wrong, how to see it +// again, and only then the environment it went wrong in. A reader who already +// knows the bug stops after the first section; a reader who says "works for +// me" is looking for the third. +func (report BugReport) Markdown() string { + var out strings.Builder + + fmt.Fprintf(&out, "# %s\n\n", report.title()) + fmt.Fprintf(&out, "In `%s`, against `%s`.\n", report.Script, report.Reference) + if tokens := report.Finding.Classification.Tokens(); len(tokens) > 0 { + fmt.Fprintf(&out, "Classified `%s`.\n", strings.Join(tokens, ", ")) + } + + out.WriteString("\n## The divergence\n\n") + out.WriteString("```\n") + out.WriteString(report.Finding.report(report.Reference, len(report.Reference))) + out.WriteString("```\n") + + // Only the first. A run that went wrong at step 5 was not really running + // the same program from step 6 onwards, and a report that lists every + // later difference is asking its reader to work out which one is the + // cause. + if later := report.Finding.StepCount - report.Finding.StepIndex - 1; later > 0 && !report.Finding.Matched { + fmt.Fprintf(&out, "\nThis is the first divergence; %s never reached.\n", pluralSteps(later)) + } + + out.WriteString("\n## Reproducing it\n\n") + out.WriteString("```sh\n") + for _, line := range report.repro() { + out.WriteString(line + "\n") + } + out.WriteString("```\n") + + out.WriteString("\n## The environment\n\n") + out.WriteString(factTable(report.Facts)) + + if len(report.TapeHeader) > 0 { + out.WriteString("\nThe tape says of itself:\n\n```\n") + for _, line := range report.TapeHeader { + out.WriteString(line + "\n") + } + out.WriteString("```\n") + } + + if report.Toolchain != "" { + out.WriteString("\nThe image was built with:\n\n```\n") + out.WriteString(report.Toolchain + "\n") + out.WriteString("```\n") + } + + return out.String() +} + +func (report BugReport) title() string { + who := strings.Join(report.Finding.Ports, ", ") + if report.Finding.Matched { + return fmt.Sprintf("%s: %s after all %d steps", who, report.Finding.Mode, report.Finding.StepCount) + } + return fmt.Sprintf("%s: %s at step %d of %d", who, report.Finding.Mode, + report.Finding.StepIndex+1, report.Finding.StepCount) +} + +// repro is the block someone else pastes. Pulling the image comes first +// because it is the step that makes the rest mean anything. +// +// Only a registry digest earns that line. A locally built image is identified +// by its config digest, a bare sha256: with no name in front of it, which +// says which bytes ran but is not something anybody else can pull -- so it is +// named in the table and left out of the commands. A command that names an +// image nobody can fetch is worse than no command. +func (report BugReport) repro() []string { + var lines []string + for _, fact := range report.Facts { + if fact.Name == "image" && strings.Contains(fact.Value, "@sha256:") { + lines = append(lines, "make pull_image PUBLISHED_REF="+fact.Value) + } + } + return append(lines, strings.Join(report.Command, " ")) +} + +// factTable is a two-column table without a header row, because both columns +// are labelled by what is in them and a "name | value" header adds a line +// nobody reads. GitHub needs the separator regardless. +func factTable(facts []Fact) string { + var table strings.Builder + table.WriteString("| | |\n| --- | --- |\n") + for _, fact := range facts { + fmt.Fprintf(&table, "| %s | %s |\n", fact.Name, inlineCode(fact.Value)) + } + return table.String() +} + +// inlineCode quotes a value as code so that a path, a digest or a version +// survives being rendered. A value with a backtick in it is left alone rather +// than escaped: nothing here has one, and a broken table is easier to notice +// than a silently mangled value. +func inlineCode(value string) string { + if value == "" || strings.ContainsRune(value, '`') { + return value + } + return "`" + value + "`" +} + +// WriteBugReport writes the report for these outcomes, if there is one to +// write. It reports whether it wrote anything, and any error is the caller's +// to report: a bug report that could not be written is worth saying out loud, +// because the run it described has already finished and will not be repeated +// for free. +func WriteBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome, commandLine []string) (bool, error) { + report, found := NewBugReport(config, runs, outcomes, commandLine) + if !found { + return false, nil + } + + if err := os.WriteFile(config.BugReport, []byte(report.Markdown()), 0o644); err != nil { + return false, err + } + return true, nil +} diff --git a/internal/play/bug_report_test.go b/internal/play/bug_report_test.go new file mode 100644 index 0000000..dad6107 --- /dev/null +++ b/internal/play/bug_report_test.go @@ -0,0 +1,281 @@ +package play + +import ( + "os" + "path/filepath" + "strings" + "testing" + + cfg "github.com/rjnienaber/prescript/internal/config" + "github.com/rjnienaber/prescript/internal/script" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func reportConfig() cfg.PlayConfig { + return cfg.PlayConfig{ScriptFile: "dice.yaml", Terminal: "pipes"} +} + +// A report about the second thing that went wrong is a report about a +// consequence. Everything after the first divergence was produced by a program +// that had already stopped running the same script. +func TestOnlyTheFirstDivergenceIsReported(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby"), comparableRun("python")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("python", atStep(1, "7 807")), + divergedOutcome("ruby", atStep(0, "HOW MANY ROLLS?")), + } + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, []string{"ruby"}, report.Finding.Ports) + assert.Contains(t, report.Markdown(), "# ruby: no-match at step 1 of 2") + + divergence := section(t, report.Markdown(), "## The divergence") + assert.Contains(t, divergence, "step 1 of 2") + assert.NotContains(t, divergence, "python") + assert.NotContains(t, divergence, "807") +} + +// section is the body of one heading. Scoped, because the rest of the report +// quotes whatever the machine happens to be -- an image digest is hex long +// enough to contain any three digits, and the image manifest names every +// language installed -- so a NotContains over the whole file asserts something +// about this machine rather than about the report. +func section(t *testing.T, markdown string, heading string) string { + t.Helper() + + _, body, found := strings.Cut(markdown, heading) + require.True(t, found, heading) + + if next := strings.Index(body, "\n## "); next >= 0 { + return body[:next] + } + return body +} + +func TestBothSidesAndTheClassificationAreInTheReport(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + matchedOutcome("basic"), + divergedOutcome("ruby", atStep(1, "7 807")), + } + + markdown := markdownFor(t, reportConfig(), runs, outcomes) + + assert.Contains(t, markdown, "Classified `formatting`") + assert.Contains(t, markdown, " basic \" 7 807 \"") + assert.Contains(t, markdown, " ruby \"7 807\"") +} + +// With one run there is no implementation being treated as correct, and +// naming one that was never played would be a lie about where the expected +// side came from. +func TestASingleRunIsReportedAgainstItsTranscript(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("ruby")} + outcomes := []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))} + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, "the transcript", report.Reference) + assert.Contains(t, report.Markdown(), "against `the transcript`") +} + +// A reference that no longer matches its own recording is the more urgent +// finding, not a reason to write nothing: the environment has moved under a +// script that used to pass. +func TestAReferenceThatDivergedIsItselfTheReport(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{ + divergedOutcome("basic", atStep(1, "7 806")), + matchedOutcome("ruby"), + } + + report, found := NewBugReport(reportConfig(), runs, outcomes, nil) + require.True(t, found) + + assert.Equal(t, []string{"basic"}, report.Finding.Ports) + assert.Equal(t, "the transcript", report.Reference) +} + +func TestThereIsNothingToReportWhenEveryRunMatched(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("basic"), comparableRun("ruby")} + outcomes := []Outcome{matchedOutcome("basic"), matchedOutcome("ruby")} + + _, found := NewBugReport(reportConfig(), runs, outcomes, nil) + assert.False(t, found) +} + +// The repro line is the command that was actually run, minus the two things +// that are about this machine rather than about the bug: how the binary was +// reached, and where the checkout happens to live. +func TestTheReproCommandIsPasteable(t *testing.T) { + t.Parallel() + + working, err := os.Getwd() + require.NoError(t, err) + + runs := []script.Run{comparableRun("ruby")} + outcomes := []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))} + commandLine := []string{"/Users/someone/go/bin/prescript", "play", + filepath.Join(working, "dice.yaml"), "--tape", "/elsewhere/dice.tape"} + + report, found := NewBugReport(reportConfig(), runs, outcomes, commandLine) + require.True(t, found) + + assert.Equal(t, + []string{"prescript", "play", "dice.yaml", "--tape", "/elsewhere/dice.tape"}, + report.Command) + assert.Contains(t, report.Markdown(), "prescript play dice.yaml --tape /elsewhere/dice.tape") +} + +func TestAnUnpinnedRunSaysSoRatherThanLeavingAGap(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| image | `none; this did not run in the pinned environment` |") + assert.NotContains(t, markdown, "make pull_image") +} + +// A digest is the only thing that makes the rest of the report reproducible, +// so when there is one it becomes the first line of the repro block. +func TestAPinnedRunPullsTheImageFirst(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "ghcr.io/rjnienaber/prescript-env@sha256:abc123") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, + "make pull_image PUBLISHED_REF=ghcr.io/rjnienaber/prescript-env@sha256:abc123") +} + +// A locally built image has a config digest and no name anybody else can pull +// it by. It says which bytes ran, which is worth recording, and it is not a +// command anyone can paste. +func TestALocallyBuiltImageIsNamedButNotPulled(t *testing.T) { + t.Setenv("PRESCRIPT_IMAGE", "sha256:9f8e7d") + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| image | `sha256:9f8e7d` |") + assert.NotContains(t, markdown, "make pull_image") +} + +// Without the tape the finding is not reproducible even in principle, so what +// the tape says about where it came from is quoted rather than summarised. +func TestTheTapeAndItsProvenanceAreQuoted(t *testing.T) { + t.Parallel() + + tape := filepath.Join(t.TempDir(), "dice.tape") + require.NoError(t, os.WriteFile(tape, + []byte("# recorded from vintbas 0.4.0, seed 0\n# 3 values\n0.5\n0.25\n0.75\n"), 0o600)) + + config := reportConfig() + config.Tape = tape + + markdown := markdownFor(t, config, []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| tape | `"+tape+"` |") + assert.Contains(t, markdown, "# recorded from vintbas 0.4.0, seed 0") + assert.Contains(t, markdown, "# 3 values") + assert.NotContains(t, markdown, "0.25") +} + +// Without a tape the seed is what makes the run repeatable, and it belongs to +// the run rather than to prescript. +func TestTheSeedStandsInForTheTape(t *testing.T) { + t.Parallel() + + run := comparableRun("ruby") + run.Env = map[string]string{"PRESCRIPT_SEED": "42", "TZ": "UTC"} + + markdown := markdownFor(t, reportConfig(), []script.Run{run}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| seed | `42` |") + assert.Contains(t, markdown, "| TZ | `UTC` |") +} + +func TestTheDrawCountIsReportedOnlyWhenItIsKnown(t *testing.T) { + t.Parallel() + + runs := []script.Run{comparableRun("ruby")} + outcome := divergedOutcome("ruby", atStep(1, "7 807")) + + assert.NotContains(t, markdownFor(t, reportConfig(), runs, []Outcome{outcome}), "| draws |") + + outcome.Draws = drew(12) + assert.Contains(t, markdownFor(t, reportConfig(), runs, []Outcome{outcome}), "| draws | `12` |") +} + +func TestTheTerminalAndPrescriptsOwnVersionAreReported(t *testing.T) { + t.Parallel() + + markdown := markdownFor(t, reportConfig(), []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}) + + assert.Contains(t, markdown, "| terminal | `pipes` |") + assert.Contains(t, markdown, "| prescript |") + assert.Contains(t, markdown, "| timeout |") +} + +// A program named by a path is the thing under test. Running somebody else's +// program with an argument it never expected is not a thing to do behind their +// back, and the answer would be the port's version rather than the toolchain's +// anyway. +func TestOnlyAnExecutableOnThePathIsAskedForItsVersion(t *testing.T) { + t.Parallel() + + assert.Empty(t, toolchainVersion("/usr/bin/ruby")) + assert.Empty(t, toolchainVersion("./dice")) + assert.Empty(t, toolchainVersion("")) + assert.Empty(t, toolchainVersion("prescript-no-such-executable")) +} + +func TestTheReportIsWrittenToTheNamedFile(t *testing.T) { + t.Parallel() + + config := reportConfig() + config.BugReport = filepath.Join(t.TempDir(), "report.md") + + written, err := WriteBugReport(config, []script.Run{comparableRun("ruby")}, + []Outcome{divergedOutcome("ruby", atStep(1, "7 807"))}, nil) + require.NoError(t, err) + assert.True(t, written) + + contents, err := os.ReadFile(config.BugReport) + require.NoError(t, err) + assert.Contains(t, string(contents), "# ruby: no-match at step 2 of 2") +} + +func markdownFor(t *testing.T, config cfg.PlayConfig, runs []script.Run, outcomes []Outcome) string { + t.Helper() + + report, found := NewBugReport(config, runs, outcomes, nil) + require.True(t, found) + + markdown := report.Markdown() + // Every section is required: a report missing one of them is the kind of + // report that gets answered with "works for me". + for _, heading := range []string{"## The divergence", "## Reproducing it", "## The environment"} { + assert.True(t, strings.Contains(markdown, heading), heading) + } + return markdown +} diff --git a/internal/play/runner_integration_test.go b/internal/play/runner_integration_test.go index 5903bda..a326a33 100644 --- a/internal/play/runner_integration_test.go +++ b/internal/play/runner_integration_test.go @@ -337,3 +337,40 @@ func TestTheDrawCountSurvivesARunThatIsKilled(t *testing.T) { }) } } + +// The report end to end, against a language that is actually installed: the +// facts a unit test has to stub -- what the interpreter says it is, how much +// of the tape the run drew, what the tape says about where it came from -- are +// exactly the ones that make a report answerable, so at least once they are +// gathered from a real run rather than described. +func TestABugReportGathersTheFactsFromARealRun(t *testing.T) { + t.Parallel() + runner := loadRunner(t, "ruby.yaml") + + // The reference's own draws, minus one, so the run diverges on the last + // line having drawn the whole tape's worth first. + wrong := strings.Replace(referenceDraws, "973", "972", 1) + runs := script.ApplyRunner([]script.Run{drawingRun(t, "rng.rb", wrong)}, runner) + + config := tapedConfig(t) + config.ScriptFile = "rng.yaml" + config.BugReport = filepath.Join(t.TempDir(), "report.md") + + outcome := Run(config, runs[0], &utils.CustomLogger{}) + assert.NotEqual(t, 0, outcome.ExitCode) + + written, err := WriteBugReport(config, runs, []Outcome{outcome}, + []string{"/somewhere/prescript", "play", "rng.yaml", "--tape", tapePath(t)}) + assert.NoError(t, err) + assert.True(t, written) + + contents, err := os.ReadFile(config.BugReport) + assert.NoError(t, err) + report := string(contents) + + assert.Contains(t, report, "| ruby | `ruby ") + assert.Contains(t, report, "| draws | `5` |") + assert.Contains(t, report, "# source: vintage-basic") + assert.Contains(t, report, "prescript play rng.yaml --tape "+tapePath(t)) + assert.Contains(t, report, referenceDraws) +} diff --git a/internal/play/runs.go b/internal/play/runs.go index 8108d3a..fd60f81 100644 --- a/internal/play/runs.go +++ b/internal/play/runs.go @@ -50,9 +50,33 @@ func RunAll(config cfg.PlayConfig, runs []script.Run, logger utils.Logger) int { fmt.Fprint(os.Stderr, Compare(runs, outcomes).Report()) } + if config.BugReport != "" { + writeBugReport(config, runs, outcomes) + } + return result } +// writeBugReport is best-effort and says so either way. It runs after every +// run has finished, so a failure here cannot be retried without paying for the +// whole script again -- which is exactly why it is reported rather than +// swallowed, and why it does not change the exit code: the runs already +// decided that, and a bug report that could not be filed is not a run that +// passed or failed differently. +func writeBugReport(config cfg.PlayConfig, runs []script.Run, outcomes []Outcome) { + // os.Args rather than the parsed config, because the point of the repro + // line is to be the command that was actually run. + written, err := WriteBugReport(config, runs, outcomes, os.Args) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "could not write bug report %s: %s\n", config.BugReport, err) + case written: + fmt.Fprintf(os.Stderr, "\nbug report written to %s\n", config.BugReport) + default: + fmt.Fprintf(os.Stderr, "\nnothing to report: no run diverged\n") + } +} + // warnAboutOverrides is a warning rather than a refusal. Overriding one thing // for every run is meaningful when the runs differ in their steps, and a // mistake when they differ in their executable, and prescript cannot tell which diff --git a/internal/utils/pty_unix.go b/internal/utils/pty_unix.go index 787b98b..acaadfd 100644 --- a/internal/utils/pty_unix.go +++ b/internal/utils/pty_unix.go @@ -11,20 +11,6 @@ import ( "golang.org/x/term" ) -// The size every one of these programs was written for, and the size the pty -// is given whatever terminal prescript itself was started from. -// -// A pty opened without a size starts at zero rows and columns, so a program -// that asks how wide its terminal is -- to decide how many columns of a table -// to print, or where to wrap -- gets an answer that is neither what a real -// terminal would give it nor what the next machine's terminal would. Either -// way that is a difference in output with no difference in behaviour behind -// it, which is exactly the kind of noise a comparison run has to be free of. -const ( - terminalColumns = 80 - terminalRows = 24 -) - // startPty starts cmd with a pseudo-terminal on all three of its standard // streams and returns this process's end of it. // diff --git a/internal/utils/terminal_size.go b/internal/utils/terminal_size.go new file mode 100644 index 0000000..34f02c5 --- /dev/null +++ b/internal/utils/terminal_size.go @@ -0,0 +1,24 @@ +package utils + +// The size every one of these programs was written for, and the size the pty +// is given whatever terminal prescript itself was started from. +// +// A pty opened without a size starts at zero rows and columns, so a program +// that asks how wide its terminal is -- to decide how many columns of a table +// to print, or where to wrap -- gets an answer that is neither what a real +// terminal would give it nor what the next machine's terminal would. Either +// way that is a difference in output with no difference in behaviour behind +// it, which is exactly the kind of noise a comparison run has to be free of. +const ( + terminalColumns = 80 + terminalRows = 24 +) + +// TerminalSize is the size prescript gives every child's pty, in columns and +// rows. Exported because a bug report has to name it: a program that wraps its +// output at the terminal's width prints something different at a different +// width, and a reader who cannot see which width was used cannot tell that +// from a real difference. +func TerminalSize() (int, int) { + return terminalColumns, terminalRows +} diff --git a/internal/utils/version.go b/internal/utils/version.go new file mode 100644 index 0000000..595c1c7 --- /dev/null +++ b/internal/utils/version.go @@ -0,0 +1,57 @@ +package utils + +import ( + "runtime/debug" +) + +// version can be stamped in at build time with +// -ldflags "-X github.com/rjnienaber/prescript/internal/utils.version=1.2.3". +// Left empty by an ordinary build, which falls back to what the toolchain +// records for itself. +var version = "" + +// Version identifies the prescript that produced a report. +// +// A bug report that does not say which build made it cannot be answered: the +// first question about any cross-implementation report is whether the tool +// still behaves that way, and the second is which tool. The Go toolchain +// stamps the commit and whether the tree was dirty into every binary built +// from a repository, so this is available without a release process to +// maintain -- and a build made from uncommitted changes says so, which is the +// case most worth knowing about. +func Version() string { + if version != "" { + return version + } + + info, available := debug.ReadBuildInfo() + if !available { + return "unknown" + } + + revision := "" + modified := false + for _, setting := range info.Settings { + switch setting.Key { + case "vcs.revision": + revision = setting.Value + case "vcs.modified": + modified = setting.Value == "true" + } + } + + if revision == "" { + if info.Main.Version != "" { + return info.Main.Version + } + return "unknown" + } + + if len(revision) > 12 { + revision = revision[:12] + } + if modified { + return revision + " (modified)" + } + return revision +}