From 9daff0a88cad7248c33afe163112996d4fddc2c7 Mon Sep 17 00:00:00 2001 From: Richard Nienaber Date: Mon, 7 Sep 2026 15:24:21 +0100 Subject: [PATCH] pin the environment a comparison run happens in Randomness is the loudest source of transcript churn but not the largest. Interpreter versions, locale, timezone, terminal size, hash seeds and address layout all change what a program prints without changing what it does, and a difference between two ports means nothing if the two ran in different worlds. docker/ builds that world. The base image is pinned by digest; Go, Node and vintbas are fetched from upstream and checked against a published SHA-256; Ruby and Python come from Debian stable, which is what pinning means for a distribution. GOTOOLCHAIN=local is in there for the same reason as the rest -- without it the go command fetches a newer toolchain when a module asks for one, quietly unpinning the Go above it. amd64 only, deliberately. The reference interpreter is published for linux x86_64, and comparing a port against a reference built for another architecture folds floating-point and standard-library differences into every comparison. Terminal size turned out to be prescript's to fix rather than the image's. A pty opened without a size reports zero rows and columns, so a program asking how wide its terminal is got an answer that was neither what a real terminal gives nor what the next machine gives. Every child now gets 80x24 whatever terminal prescript itself was started from. Address-space randomisation is left on by default. Docker's default seccomp profile permits personality() with a handful of arguments and ADDR_NO_RANDOMIZE is not among them, so holding the layout still means dropping the profile entirely -- worth keeping until a run actually needs it, given the code being compared is other people's and the repository is mounted writable. PRESCRIPT_FIX_ADDRESS_LAYOUT=1 asks for it, and the entrypoint says on stderr when addresses will move. What a bug report needs is readable from inside a run: PRESCRIPT_IMAGE names the image, and $PRESCRIPT_ENVIRONMENT names a file listing the base digest and every toolchain version, written at build time from the binaries that were actually installed rather than from the arguments that asked for them. Issue #18 folds both into a repro block. CI builds the image and runs the suite inside it on every change, because an image nobody builds is an image that has stopped building. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019f9gkZVUEo3jtv51GSb4vJ --- .github/workflows/ci.yaml | 16 ++++ Makefile | 25 ++++++ README.md | 10 +++ docker/Dockerfile | 111 +++++++++++++++++++++++++ docker/entrypoint.sh | 25 ++++++ docker/run.sh | 60 +++++++++++++ docs/container.md | 104 +++++++++++++++++++++++ docs/determinism.md | 8 +- internal/play/play_integration_test.go | 15 ++++ internal/utils/pty_unix.go | 18 ++++ test/fixtures/terminal_size.sh | 5 ++ 11 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 docker/Dockerfile create mode 100755 docker/entrypoint.sh create mode 100755 docker/run.sh create mode 100644 docs/container.md create mode 100755 test/fixtures/terminal_size.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f6b248a..b6b4b8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -48,6 +48,22 @@ jobs: - name: make examples run: make examples + # The pinned environment, built and then used. A comparison between two + # ports means nothing if the two ran in different worlds, so the image a + # developer runs and the image CI runs have to be the one image -- and the + # only way to know it still builds is to build it on every change. + container: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v7 + + - name: make image + run: make image + + - name: make docker_test + run: make docker_test + # Formatting and linting are properties of the repository, not of any one # Go version, so they run once on the newest. checks: diff --git a/Makefile b/Makefile index 6845876..bb41302 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ GCI_VERSION := v0.14.0 +IMAGE ?= prescript-env:dev VINTBAS_RELEASE := vintbas-1.0.3-1 VINTBAS_ASSET := vintbas-$(shell uname -s | tr '[:upper:]' '[:lower:]')-$(shell uname -m) VINTBAS_URL := https://github.com/rjnienaber/prescript/releases/download/$(VINTBAS_RELEASE)/$(VINTBAS_ASSET) @@ -31,6 +32,30 @@ vintbas: tape: ./scripts/record-tape.sh +# The pinned environment. Interpreter versions, locale, timezone, terminal +# size, hash seeds and address layout all change what a program prints without +# changing what it does, so a difference between two ports only means something +# when both ran in here. See docs/container.md. +.PHONY: image +image: + docker build --platform linux/amd64 -t $(IMAGE) docker + +# The same checks CI runs, run in the same image CI runs them in. +.PHONY: docker_test +docker_test: + ./docker/run.sh make container_checks + +.PHONY: docker_shell +docker_shell: + ./docker/run.sh bash + +# Everything that needs an interpreter, and nothing that needs the lint and +# format tooling: those are properties of the repository rather than of the +# environment, and pinning a linter into the image would mean rebuilding it to +# upgrade one. +.PHONY: container_checks +container_checks: test build_dev examples + .PHONY: format format: go fmt ./... diff --git a/README.md b/README.md index c9b269e..d823de5 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,9 @@ pty takes anything the program started with it. ``` make dependencies # build tooling, plus the vintbas interpreter the examples run against make prepush # format, lint, test, build, examples + +make image # the pinned environment +make docker_test # the suite, inside it ``` `make examples` runs a BASIC program under Vintage BASIC, which seeds its random @@ -281,6 +284,13 @@ package pinned in the build script: `random` 1.2 replaced the generator behind `StdGen`, so the same seed yields different numbers either side of it. Changing the pin invalidates any recorded output. +Randomness is not the only thing that moves. Interpreter versions, locale, +timezone, terminal size, hash seeds and address layout change what a program +prints without changing what it does, and a difference between two ports means +nothing if the two ran in different worlds. `docker/` holds a pinned +environment that fixes all of them, used by CI and available locally; see +[docs/container.md](docs/container.md). + ### Get in touch * Slack: Find me as @rjnienaber on https://gophers.slack.com/ * Twitter: [@rjnienaber](https://twitter.com/rjnienaber) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..f59363b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,111 @@ +# The environment a comparison run happens in. +# +# Randomness is the loudest source of transcript churn but not the largest. +# Interpreter versions, locale, timezone, terminal size, hash seeds and address +# layout all change what a program prints without changing what it does, and a +# difference between two ports means nothing if the two ran in different +# worlds. A seed and a tape make the numbers agree; this makes everything +# around them agree. +# +# Everything is pinned by digest or checksum, so `docker build` produces the +# same toolchains today and next year. The pins are the point -- an image that +# drifts is an image that turns a fixed bug back into an open one. +# +# amd64 only, and deliberately: the reference interpreter is published for +# linux x86_64, and comparing against a reference built for another +# architecture would fold floating-point and library differences into every +# comparison. Building this on arm64 works through emulation and is slow. + +ARG BASE_DIGEST=sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 +FROM --platform=linux/amd64 debian:bookworm-slim@${BASE_DIGEST} +ARG BASE_DIGEST + +# Ruby and Python come from Debian stable, which is what pinning means for a +# distribution: a fixed series that takes security fixes and no new features +# for the life of the release. Go, Node and the reference interpreter are not +# in it, so they are fetched from upstream and checked against a published +# checksum before anything is unpacked. +ARG GO_VERSION=1.24.13 +ARG GO_SHA256=1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730 +ARG NODE_VERSION=24.20.0 +ARG NODE_SHA256=2f2c0da162318f0de47665410c7c8c2ed3d36c8f3105de4bbc61176c70a7cbf2 +ARG VINTBAS_RELEASE=vintbas-1.0.3-1 +ARG VINTBAS_SHA256=72dda1d16ed21f0d812f6978a32d105dadea49dd8163692b44c462678eb16a17 + +SHELL ["/bin/bash", "-o", "pipefail", "-euc"] + +RUN apt-get update \ + && apt-get install --no-install-recommends --yes \ + ca-certificates \ + curl \ + git \ + make \ + python3 \ + ruby \ + tzdata \ + util-linux \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL -o /tmp/go.tar.gz "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + && echo "${GO_SHA256} /tmp/go.tar.gz" | sha256sum --check --strict - \ + && tar --directory /usr/local --extract --file /tmp/go.tar.gz \ + && rm /tmp/go.tar.gz + +RUN curl -fsSL -o /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + && echo "${NODE_SHA256} /tmp/node.tar.xz" | sha256sum --check --strict - \ + && tar --directory /usr/local --extract --file /tmp/node.tar.xz --strip-components 1 \ + --exclude CHANGELOG.md --exclude LICENSE --exclude README.md \ + && rm /tmp/node.tar.xz + +# The reference BASIC interpreter, patched to seed its RNG from VINTBAS_SEED. +# The same binary the Makefile installs, so a transcript recorded here and one +# recorded on a developer's machine came from the same interpreter. +RUN curl -fsSL -o /usr/local/bin/vintbas \ + "https://github.com/rjnienaber/prescript/releases/download/${VINTBAS_RELEASE}/vintbas-linux-x86_64" \ + && echo "${VINTBAS_SHA256} /usr/local/bin/vintbas" | sha256sum --check --strict - \ + && chmod +x /usr/local/bin/vintbas + +ENV PATH=/usr/local/go/bin:/usr/local/bin:$PATH + +# The rest of what moves. Each of these has been a transcript diff at some +# point in somebody's life: +# +# LANG/LC_ALL decimal separators, sort order, and how a byte over 127 prints +# TZ any program that prints a date +# COLUMNS a program that wraps its own output; belt and braces with the +# fixed pty size prescript gives the child +# PYTHONHASHSEED the iteration order of a set or a dict of strings +# GOTOOLCHAIN stops the go command downloading a different Go than the one +# pinned above, which would quietly unpin the image +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + TZ=UTC \ + COLUMNS=80 \ + LINES=24 \ + PYTHONHASHSEED=0 \ + PYTHONDONTWRITEBYTECODE=1 \ + GOTOOLCHAIN=local + +# What a bug report has to carry, written from the binaries that were actually +# installed rather than from the arguments above, so it cannot claim a version +# the image does not have. Issue #18 reads this file; a human can cat it. +ENV PRESCRIPT_ENVIRONMENT=/etc/prescript-environment +RUN { \ + echo "base=debian:bookworm-slim@${BASE_DIGEST}"; \ + echo "arch=$(uname -m)"; \ + echo "go=$(go version | cut -d' ' -f3)"; \ + echo "node=$(node --version)"; \ + echo "python=$(python3 --version | cut -d' ' -f2)"; \ + echo "ruby=$(ruby --version | cut -d' ' -f2)"; \ + echo "vintbas=${VINTBAS_RELEASE}"; \ + echo "locale=${LC_ALL}"; \ + echo "timezone=${TZ}"; \ + } > "${PRESCRIPT_ENVIRONMENT}" + +COPY entrypoint.sh /usr/local/bin/prescript-entrypoint +RUN chmod +x /usr/local/bin/prescript-entrypoint + +WORKDIR /workspace +ENTRYPOINT ["/usr/local/bin/prescript-entrypoint"] +CMD ["bash"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..f2d1450 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Turns off address-space randomisation for everything run in the container. +# +# ASLR is the one source of churn that no environment variable reaches. A +# runtime that folds a pointer address into a hash or an RNG seed -- and +# several do, as a defence against collision attacks -- prints something +# different on every run of an otherwise identical program, which looks exactly +# like the divergence this tool exists to find. +# +# Best effort on purpose, and usually refused: Docker's default seccomp profile +# permits personality() with a handful of arguments and ADDR_NO_RANDOMIZE is +# not among them, so holding the address layout still means dropping the +# profile. docker/run.sh does that when PRESCRIPT_FIX_ADDRESS_LAYOUT=1 asks it +# to. Refusal is not fatal here -- a container that will not start is worse +# than one that is slightly less repeatable -- and the warning goes to stderr +# so it lands in the log rather than in a transcript. +set -euo pipefail + +if setarch --addr-no-randomize true 2> /dev/null; then + exec setarch --addr-no-randomize "$@" +fi + +echo "prescript: address-space randomisation is on, so a runtime that mixes pointers into a hash or a seed will vary between runs; set PRESCRIPT_FIX_ADDRESS_LAYOUT=1 to turn it off" >&2 +exec "$@" diff --git a/docker/run.sh b/docker/run.sh new file mode 100755 index 0000000..9e11c64 --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Runs a command inside the pinned environment, with the repository mounted. +# +# docker/run.sh make test +# docker/run.sh bash +# +# Everything the container needs to be reproducible is set here rather than +# left to whoever types the docker command, because a run that forgot one of +# these flags looks exactly like a run that did not. +# +# The repository is mounted so a change can be tried without a rebuild, but +# tmp/ is not: `make build_dev` in here produces a linux binary, and dropping +# that on top of the one the host just built turns the next `make examples` +# into "cannot execute binary file". +set -euo pipefail + +IMAGE="${PRESCRIPT_IMAGE_NAME:-prescript-env:dev}" +REPOSITORY="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if ! docker image inspect "$IMAGE" > /dev/null 2>&1; then + echo "$IMAGE has not been built; run 'make image' first" >&2 + exit 1 +fi + +# What a bug report has to name alongside the versions inside the image: which +# image. A published image is identified by its registry digest and a locally +# built one by its config digest, and either is enough for someone else to get +# the same bytes. +DIGEST="$(docker image inspect \ + --format '{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}' "$IMAGE")" + +# Docker's default seccomp profile permits personality() with a handful of +# arguments and ADDR_NO_RANDOMIZE is not among them, so fixing the address +# layout means dropping the profile entirely. That is off by default and asked +# for explicitly, because the code being compared is other people's and the +# repository is mounted writable: the profile is worth keeping until a run +# actually needs the address layout held still. +SECCOMP=() +if [ "${PRESCRIPT_FIX_ADDRESS_LAYOUT:-0}" = "1" ]; then + SECCOMP=(--security-opt seccomp=unconfined) +fi + +# A tty only when there is one to pass on, so this works the same from a +# terminal and from CI. Written the long way because macOS still ships bash +# 3.2, where an empty array under `set -u` is an error rather than nothing. +INTERACTIVE=() +if [ -t 0 ]; then + INTERACTIVE=(--interactive --tty) +fi + +exec docker run --rm ${INTERACTIVE[@]+"${INTERACTIVE[@]}"} ${SECCOMP[@]+"${SECCOMP[@]}"} \ + --platform linux/amd64 \ + --volume "$REPOSITORY:/workspace" \ + --mount type=volume,source=prescript-tmp,target=/workspace/tmp \ + --mount type=volume,source=prescript-go-build,target=/root/.cache/go-build \ + --mount type=volume,source=prescript-go-mod,target=/root/go/pkg/mod \ + --env PRESCRIPT_IMAGE="$DIGEST" \ + --env PRESCRIPT_REQUIRE_RUNNERS=1 \ + "$IMAGE" "$@" diff --git a/docs/container.md b/docs/container.md new file mode 100644 index 0000000..d04687c --- /dev/null +++ b/docs/container.md @@ -0,0 +1,104 @@ +# The pinned environment + +A difference between two ports only means something when both ran in the same +world. [Determinism](determinism.md) covers the randomness half of that — a +seed makes one port repeatable, a tape makes every port draw the same numbers. +This is the other half, and in practice the larger one: interpreter versions, +locale, timezone, terminal size, hash seeds and address layout all change what +a program prints without changing what it does. + +``` +make image # build it +make docker_test # run the suite inside it +make docker_shell # look around +``` + +## What is pinned + +| | How | +| --- | --- | +| Base | `debian:bookworm-slim` by digest | +| Ruby, Python | Debian stable, a fixed series taking security fixes only | +| Go, Node | upstream tarballs, checked against a published SHA-256 | +| vintbas | the same release asset the Makefile installs, checked the same way | +| Locale | `LANG=LC_ALL=C.UTF-8` | +| Timezone | `TZ=UTC` | +| Terminal | 80×24, set on the pty prescript gives the child | +| Hash seed | `PYTHONHASHSEED=0` | +| Address layout | `setarch --addr-no-randomize`, opt-in; see below | + +`GOTOOLCHAIN=local` is in there for the same reason as everything else: without +it the `go` command will fetch a newer toolchain when a module asks for one, +which would quietly unpin the Go that was so carefully pinned above. + +Rebuilding produces the same toolchains next year as today. That is the whole +point — an image that drifts turns a fixed bug back into an open one. + +## amd64 only, deliberately + +The reference interpreter is published for linux x86_64. Comparing a port +against a reference built for a different architecture folds floating-point and +standard-library differences into every comparison, which is exactly the noise +this image exists to remove. Building on an arm64 machine works through +emulation and is slow; running the tests that way is fine, recording a tape +that way is not. + +## Terminal size is prescript's job, not the image's + +`COLUMNS` and `LINES` are set here, but the number that actually matters is the +one the kernel reports for the pty, and that is prescript's to set. It gives +every child an 80×24 pty whatever terminal prescript itself was started from — +a pty opened without a size reports zero rows and columns, and a program asking +how wide its terminal is would otherwise get an answer that is neither what a +real terminal gives nor what the next machine gives. + +## What a bug report needs + +Two things, and both are readable from inside a run: + +- `$PRESCRIPT_ENVIRONMENT` names a file listing the base digest and every + toolchain version. It is written at build time from the binaries that were + actually installed, not from the arguments that asked for them, so it cannot + claim a version the image does not have. +- `$PRESCRIPT_IMAGE` is the image itself — its registry digest when it was + pulled, its config digest when it was built locally. `docker/run.sh` sets it, + because an image can not know its own digest at build time. + +``` +$ make docker_shell +# cat $PRESCRIPT_ENVIRONMENT +base=debian:bookworm-slim@sha256:8820... +arch=x86_64 +go=go1.24.13 +node=v24.20.0 +python=3.11.2 +ruby=3.1.2p20 +vintbas=vintbas-1.0.3-1 +locale=C.UTF-8 +timezone=UTC +``` + +Issue #18 folds both into the repro block of a generated bug report. + +## What it does not do + +The image is not published anywhere yet, so CI and a developer each build it +from the same pinned Dockerfile rather than pulling the same bytes. That is +enough to make the toolchains agree and not enough to make the image layers +byte-identical; publishing it is the obvious follow-up. + +Address-space randomisation is left on by default. Docker's default seccomp +profile permits `personality()` with a handful of arguments and +`ADDR_NO_RANDOMIZE` is not among them, so holding the layout still means +dropping the profile entirely — and the code being compared is other people's, +run against a writable mount of the repository, so the profile is worth keeping +until a run actually needs the addresses held still: + +``` +PRESCRIPT_FIX_ADDRESS_LAYOUT=1 make docker_test +``` + +Without it the entrypoint says on stderr that addresses will move, and carries +on. A container that will not start is worse than one that is slightly less +repeatable, and this matters only for a runtime that mixes a pointer into a +hash or a seed. diff --git a/docs/determinism.md b/docs/determinism.md index 9b7e16f..7378686 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -104,8 +104,8 @@ sitting and identical on every platform. The first two are runners waiting for a machine to test them on. The last three are a different problem: a compiled language's randomness is decided when it is built, and a runner only gets to speak at launch. Those need the build to -cooperate, which is what makes the pinned image in the container work -(issue #15) the place they belong. +cooperate, which is what makes the [pinned image](container.md) the place they +belong. An untested runner is worse than a missing one — it makes a corpus look seeded when it is not — so these are listed rather than guessed at. @@ -229,8 +229,8 @@ Transcripts also diverge on locale, timezone, terminal size, hash seeds and address-space layout, and in practice those cause more churn than the RNG does. `runners/python.yaml` sets `PYTHONHASHSEED=0` for that reason — a program that prints a set is otherwise a different program on every run whatever its -generator does — but the rest belongs in a pinned image rather than in a runner -(issue #15). +generator does — but the rest belongs in a pinned image rather than in a +runner, and that is what `docker/` is: see [container.md](container.md). Those runners set `inheritEnv: true`, which is a compromise and worth naming: a Ruby installed by rbenv, a Python by pyenv or a Node by nvm all find their own diff --git a/internal/play/play_integration_test.go b/internal/play/play_integration_test.go index 82e9c32..e68feff 100644 --- a/internal/play/play_integration_test.go +++ b/internal/play/play_integration_test.go @@ -319,6 +319,21 @@ func TestExecutableRunsUnderAPtyByDefault(t *testing.T) { assert.Equal(t, 0, exitCode) } +// A program asking how wide its terminal is gets the same answer here as it +// does on the next machine, whatever terminal prescript itself was started +// from -- and an answer at all, which a pty opened without a size does not +// give: that one reports zero rows and columns. +func TestThePtyIsAlwaysTheSameSize(t *testing.T) { + t.Parallel() + + config := createConfig(t, "fixtures/terminal_size.sh") + run := script.Run{Steps: []script.Step{{Line: "24 80"}}} + + exitCode := Run(config.Play, run, config.Logger) + + assert.Equal(t, 0, exitCode) +} + func TestTerminalPipesGivesTheExecutablePipes(t *testing.T) { t.Parallel() diff --git a/internal/utils/pty_unix.go b/internal/utils/pty_unix.go index 6d6616b..787b98b 100644 --- a/internal/utils/pty_unix.go +++ b/internal/utils/pty_unix.go @@ -11,6 +11,20 @@ import ( "golang.org/x/term" ) +// The size every one of these programs was written for, and the size the pty +// is given whatever terminal prescript itself was started from. +// +// A pty opened without a size starts at zero rows and columns, so a program +// that asks how wide its terminal is -- to decide how many columns of a table +// to print, or where to wrap -- gets an answer that is neither what a real +// terminal would give it nor what the next machine's terminal would. Either +// way that is a difference in output with no difference in behaviour behind +// it, which is exactly the kind of noise a comparison run has to be free of. +const ( + terminalColumns = 80 + terminalRows = 24 +) + // startPty starts cmd with a pseudo-terminal on all three of its standard // streams and returns this process's end of it. // @@ -49,6 +63,10 @@ func startPty(cmd *exec.Cmd) (*os.File, error) { return closePty(err) } + if err := pty.Setsize(ptmx, &pty.Winsize{Rows: terminalRows, Cols: terminalColumns}); err != nil { + return closePty(err) + } + cmd.Stdin, cmd.Stdout, cmd.Stderr = tty, tty, tty // Setsid and Setctty make the child a session leader with this pty as its diff --git a/test/fixtures/terminal_size.sh b/test/fixtures/terminal_size.sh new file mode 100755 index 0000000..1feaff3 --- /dev/null +++ b/test/fixtures/terminal_size.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# +# Prints the size the kernel reports for the terminal on stdin, which is what +# a program consults before deciding where to wrap its output. +stty size