From 4dc570cf8ed24f18063d16c12d2e5b854f04e208 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Thu, 27 Aug 2026 22:18:09 +0200 Subject: [PATCH 01/17] fix(process): bound context command cleanup Amp-Thread-ID: https://ampcode.com/threads/T-01a0448f-5860-721c-8a47-5119fc57f685 Co-authored-by: Amp --- internal/agenteval/agent_command.go | 3 ++ internal/agenteval/materialize.go | 3 ++ internal/agenteval/run.go | 4 ++ internal/dictation/runner.go | 3 ++ internal/execution/command_context.go | 22 +++++++++ internal/hooks/dispatch.go | 1 + internal/hooks/dispatch_test.go | 36 +++++++++++++++ internal/perfbench/perfbench.go | 64 +++++++-------------------- internal/perfbench/taskbench.go | 4 ++ internal/perfbench/turn_bench.go | 2 + internal/verify/verify.go | 2 + 11 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 internal/execution/command_context.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index a07d32ef2..dc4cf5197 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -6,6 +6,8 @@ import ( "errors" "os/exec" "strings" + + "github.com/Gitlawb/zero/internal/execution" ) type AgentRunInput struct { @@ -61,6 +63,7 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A } command := expandAgentCommand(runner.Command, input) cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = input.WorkspacePath stdout := &capWriter{limit: limit} stderr := &capWriter{limit: limit} diff --git a/internal/agenteval/materialize.go b/internal/agenteval/materialize.go index 71e683dfc..e5076eb3e 100644 --- a/internal/agenteval/materialize.go +++ b/internal/agenteval/materialize.go @@ -10,6 +10,8 @@ import ( "os/exec" "path/filepath" "strings" + + "github.com/Gitlawb/zero/internal/execution" ) type Materializer struct{} @@ -188,6 +190,7 @@ func initGitBaseline(ctx context.Context, workspace string) error { } for _, args := range commands { cmd := exec.CommandContext(ctx, "git", args...) + execution.HardenCommandContext(cmd) cmd.Dir = workspace cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=Zero Eval", diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index e6f13a500..2e8b53557 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -10,6 +10,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // defaultCommandTimeout bounds a single verification command so a hung command @@ -139,6 +141,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command } args := trimCommand(command.Command) cmd := exec.CommandContext(ctx, args[0], args[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = workspace var stdout bytes.Buffer var stderr bytes.Buffer @@ -167,6 +170,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command func defaultRunGit(ctx context.Context, workspace string, args ...string) ([]byte, error) { allArgs := append([]string{"-C", workspace}, args...) cmd := exec.CommandContext(ctx, "git", allArgs...) + execution.HardenCommandContext(cmd) output, err := cmd.Output() if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { diff --git a/internal/dictation/runner.go b/internal/dictation/runner.go index 3db6eb156..bdfe6362a 100644 --- a/internal/dictation/runner.go +++ b/internal/dictation/runner.go @@ -7,6 +7,8 @@ import ( "os" "os/exec" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // commandSpec describes one capture-process invocation. Argv is always @@ -104,6 +106,7 @@ func (p *realProcess) Kill() error { return p.cmd.Process.Kill() } func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) + execution.HardenCommandContext(cmd) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go new file mode 100644 index 000000000..7f21fa71a --- /dev/null +++ b/internal/execution/command_context.go @@ -0,0 +1,22 @@ +package execution + +import ( + "os/exec" +) + +// HardenCommandContext makes a context-bound command terminate its process +// tree and prevents inherited output handles from blocking Wait indefinitely. +// Call this before Start or Run. +func HardenCommandContext(command *exec.Cmd) { + if command == nil { + return + } + ConfigureProcessGroup(command) + command.WaitDelay = processWaitDelay + command.Cancel = func() error { + if command.Process == nil { + return nil + } + return KillProcessTree(command.Process.Pid) + } +} diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index d5bb13e3c..946ed4b3a 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -298,6 +298,7 @@ func (dispatcher *Dispatcher) recordCompleted(hook Definition, input DispatchInp // ExitCode (not Err); Err is reserved for commands that could not be launched. func execCommandRunner(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { cmd := exec.CommandContext(ctx, command, args...) + execution.HardenCommandContext(cmd) cmd.Dir = cwd cmd.Env = env if len(stdin) > 0 { diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 40d6e295f..099f3b5be 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -2,6 +2,7 @@ package hooks import ( "context" + "os" "os/exec" "path/filepath" "runtime" @@ -25,6 +26,41 @@ func beforeToolConfig(hooks ...Definition) Config { return Config{Enabled: true, Hooks: hooks} } +func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { + switch os.Getenv("ZERO_HOOK_TREE_HELPER") { + case "parent": + child := exec.Command(os.Args[0], "-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$") + child.Env = append(os.Environ(), "ZERO_HOOK_TREE_HELPER=grandchild") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + select {} + case "grandchild": + time.Sleep(30 * time.Second) + os.Exit(0) + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + result := execCommandRunner( + ctx, + os.Args[0], + []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, + nil, + "", + append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent"), + ) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) + } + if result.Err == nil && result.ExitCode == 0 { + t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) + } +} + func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { var calls []string runner := func(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { diff --git a/internal/perfbench/perfbench.go b/internal/perfbench/perfbench.go index 31afe7b69..762ee1ee9 100644 --- a/internal/perfbench/perfbench.go +++ b/internal/perfbench/perfbench.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/release" ) @@ -266,6 +267,7 @@ func MeasureColdStart(ctx context.Context, command []string) (float64, error) { } startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) output, err := cmd.CombinedOutput() durationMs := RoundMetric(float64(time.Since(startedAt).Microseconds()) / 1000) @@ -282,17 +284,9 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl rssBefore := readHarnessMemoryMb() startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Env = offlineBenchmarkEnv(os.Environ()) - stdout, err := cmd.StdoutPipe() - if err != nil { - return firstOutputSample{}, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return firstOutputSample{}, err - } - var once sync.Once var firstOutputAt time.Time markFirstOutput := func() { @@ -300,32 +294,19 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl firstOutputAt = time.Now() }) } - - if err := cmd.Start(); err != nil { - return firstOutputSample{}, err - } - stdoutChan := make(chan pipeResult, 1) - stderrChan := make(chan pipeResult, 1) - go readTimedPipe(stdout, markFirstOutput, stdoutChan) - go readTimedPipe(stderr, markFirstOutput, stderrChan) - - stdoutResult := <-stdoutChan - stderrResult := <-stderrChan - waitErr := cmd.Wait() + stdout := &timedBuffer{onFirstWrite: markFirstOutput} + stderr := &timedBuffer{onFirstWrite: markFirstOutput} + cmd.Stdout = stdout + cmd.Stderr = stderr + waitErr := cmd.Run() finishedAt := time.Now() - if stdoutResult.Err != nil { - return firstOutputSample{}, stdoutResult.Err - } - if stderrResult.Err != nil { - return firstOutputSample{}, stderrResult.Err - } if firstOutputAt.IsZero() { firstOutputAt = finishedAt } rssAfter := readHarnessMemoryMb() if waitErr != nil { - return firstOutputSample{}, commandError(command, waitErr, stdoutResult.Text, stderrResult.Text) + return firstOutputSample{}, commandError(command, waitErr, stdout.String(), stderr.String()) } return firstOutputSample{ FirstOutputMs: RoundMetric(float64(firstOutputAt.Sub(startedAt).Microseconds()) / 1000), @@ -421,29 +402,16 @@ func median(sortedSamples []float64) float64 { return RoundMetric((sortedSamples[middle-1] + sortedSamples[middle]) / 2) } -type pipeResult struct { - Text string - Err error +type timedBuffer struct { + bytes.Buffer + onFirstWrite func() } -func readTimedPipe(reader io.Reader, onFirstChunk func(), result chan<- pipeResult) { - var buffer bytes.Buffer - chunk := make([]byte, 32*1024) - for { - n, err := reader.Read(chunk) - if n > 0 { - onFirstChunk() - _, _ = buffer.Write(chunk[:n]) - } - if err != nil { - if errors.Is(err, io.EOF) { - result <- pipeResult{Text: buffer.String()} - return - } - result <- pipeResult{Text: buffer.String(), Err: err} - return - } +func (buffer *timedBuffer) Write(data []byte) (int, error) { + if len(data) > 0 { + buffer.onFirstWrite() } + return buffer.Buffer.Write(data) } func commandError(command []string, err error, stdout string, stderr string) error { diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index cfcab1a86..ea9a0d86f 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -13,6 +13,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // TaskSchemaVersion is the schema version of a published task-benchmark result. @@ -328,6 +330,7 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { return func(ctx context.Context, task BenchTask, rc RunContext) TaskOutcome { args := buildExecArgs(task, rc, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -383,6 +386,7 @@ func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { func runVerification(ctx context.Context, task BenchTask) TaskOutcome { cmd := exec.CommandContext(ctx, task.VerificationCommand[0], task.VerificationCommand[1:]...) + execution.HardenCommandContext(cmd) // Inherit the environment so the verifier sees PATH, HOME, language toolchain // vars, etc. — the same surface a maintainer gets running the command by hand // (matching the agent run above). NO_COLOR is appended for stable output. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index a157b140b..788a95944 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -16,6 +16,7 @@ import ( "time" "github.com/Gitlawb/zero/internal/execprofile" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/trace" ) @@ -657,6 +658,7 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { args := buildTurnExecArgs(task, rc, tracePath, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir diff --git a/internal/verify/verify.go b/internal/verify/verify.go index 363cad6ba..cd152bf41 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/redaction" "github.com/Gitlawb/zero/internal/testrunner" ) @@ -299,6 +300,7 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti defer cancel() cmd := exec.CommandContext(commandCtx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = dir var stdout bytes.Buffer var stderr bytes.Buffer From 8621fdb8d88fcbf00f2690bcd9fd3d0beb5980a5 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 27 Aug 2026 21:09:25 +0000 Subject: [PATCH 02/17] fix(process): retain command tree cancellation identity Amp-Thread-ID: https://ampcode.com/threads/T-01a044e2-92ad-774b-9a86-094e2e5293bf Co-authored-by: Pierre Bruno --- internal/agenteval/agent_command.go | 3 +- internal/agenteval/materialize.go | 3 +- internal/agenteval/run.go | 10 +-- internal/dictation/runner.go | 3 +- internal/execution/command_context.go | 55 ++++++++++++++--- internal/execution/command_context_test.go | 61 +++++++++++++++++++ .../execution/command_context_unix_test.go | 19 ++++++ .../execution/command_context_windows_test.go | 36 +++++++++++ internal/execution/command_tree_unix.go | 41 +++++++++++++ internal/execution/command_tree_windows.go | 60 ++++++++++++++++++ internal/hooks/dispatch.go | 3 +- internal/hooks/dispatch_test.go | 40 +++++++++--- internal/hooks/process_test_unix.go | 28 +++++++++ internal/hooks/process_test_windows.go | 36 +++++++++++ internal/perfbench/perfbench.go | 11 ++-- internal/perfbench/taskbench.go | 14 +++-- internal/perfbench/taskbench_test.go | 15 +++++ internal/perfbench/turn_bench.go | 8 ++- internal/perfbench/turn_bench_test.go | 14 +++++ internal/verify/verify.go | 7 ++- internal/verify/verify_test.go | 36 +++++++++++ 21 files changed, 456 insertions(+), 47 deletions(-) create mode 100644 internal/execution/command_context_test.go create mode 100644 internal/execution/command_context_unix_test.go create mode 100644 internal/execution/command_context_windows_test.go create mode 100644 internal/execution/command_tree_unix.go create mode 100644 internal/execution/command_tree_windows.go create mode 100644 internal/hooks/process_test_unix.go create mode 100644 internal/hooks/process_test_windows.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index dc4cf5197..8919529ea 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -63,14 +63,13 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A } command := expandAgentCommand(runner.Command, input) cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = input.WorkspacePath stdout := &capWriter{limit: limit} stderr := &capWriter{limit: limit} cmd.Stdout = stdout cmd.Stderr = stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result.Stdout = stdout.buf.String() result.Stderr = stderr.buf.String() result.Truncated = stdout.truncated || stderr.truncated diff --git a/internal/agenteval/materialize.go b/internal/agenteval/materialize.go index e5076eb3e..3ab705a4a 100644 --- a/internal/agenteval/materialize.go +++ b/internal/agenteval/materialize.go @@ -190,7 +190,6 @@ func initGitBaseline(ctx context.Context, workspace string) error { } for _, args := range commands { cmd := exec.CommandContext(ctx, "git", args...) - execution.HardenCommandContext(cmd) cmd.Dir = workspace cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=Zero Eval", @@ -201,7 +200,7 @@ func initGitBaseline(ctx context.Context, workspace string) error { var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - if err := cmd.Run(); err != nil { + if err := execution.RunCommand(ctx, cmd); err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index 2e8b53557..07a60471a 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -141,13 +141,12 @@ func execCommand(ctx context.Context, workspace string, command Command) Command } args := trimCommand(command.Command) cmd := exec.CommandContext(ctx, args[0], args[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = workspace var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result.Stdout = stdout.String() result.Stderr = stderr.String() if err == nil { @@ -170,15 +169,16 @@ func execCommand(ctx context.Context, workspace string, command Command) Command func defaultRunGit(ctx context.Context, workspace string, args ...string) ([]byte, error) { allArgs := append([]string{"-C", workspace}, args...) cmd := exec.CommandContext(ctx, "git", allArgs...) - execution.HardenCommandContext(cmd) - output, err := cmd.Output() + var output bytes.Buffer + cmd.Stdout = &output + err := execution.RunCommand(ctx, cmd) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return nil, ctxErr } return nil, err } - return output, nil + return output.Bytes(), nil } func parseGitStatusPorcelain(output []byte) []string { diff --git a/internal/dictation/runner.go b/internal/dictation/runner.go index bdfe6362a..43b250559 100644 --- a/internal/dictation/runner.go +++ b/internal/dictation/runner.go @@ -106,11 +106,10 @@ func (p *realProcess) Kill() error { return p.cmd.Process.Kill() } func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) - execution.HardenCommandContext(cmd) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) return out.Bytes(), err } diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 7f21fa71a..6e40af31a 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -1,22 +1,57 @@ package execution import ( + "context" + "errors" + "fmt" "os/exec" ) -// HardenCommandContext makes a context-bound command terminate its process -// tree and prevents inherited output handles from blocking Wait indefinitely. -// Call this before Start or Run. -func HardenCommandContext(command *exec.Cmd) { +// RunCommand runs a context-bound command in a retained process tree and +// prevents inherited output handles from blocking Wait indefinitely. +func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if command == nil { - return + return errors.New("execution: nil command") } - ConfigureProcessGroup(command) + if ctx == nil { + ctx = context.Background() + } + tree, err := prepareCommandTree(command) + if err != nil { + return err + } + defer func() { err = errors.Join(err, tree.close()) }() + command.WaitDelay = processWaitDelay - command.Cancel = func() error { - if command.Process == nil { - return nil + command.Cancel = tree.cancel + if err := command.Start(); err != nil { + _ = tree.attach(nil) + return err + } + if err := tree.attach(command.Process); err != nil { + killErr := command.Process.Kill() + waitErr := command.Wait() + return errors.Join(fmt.Errorf("execution: attach process tree: %w", err), killErr, waitErr) + } + waitComplete := make(chan struct{}) + type cancellation struct { + err error + canceled bool + } + cancelResult := make(chan cancellation, 1) + go func() { + select { + case <-ctx.Done(): + cancelResult <- cancellation{err: tree.cancel(), canceled: true} + case <-waitComplete: + cancelResult <- cancellation{} } - return KillProcessTree(command.Process.Pid) + }() + waitErr := command.Wait() + close(waitComplete) + canceled := <-cancelResult + if canceled.canceled { + return errors.Join(waitErr, ctx.Err(), canceled.err) } + return waitErr } diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go new file mode 100644 index 000000000..7c0c052da --- /dev/null +++ b/internal/execution/command_context_test.go @@ -0,0 +1,61 @@ +package execution + +import ( + "bytes" + "context" + "os" + "os/exec" + "strconv" + "testing" + "time" +) + +func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { + switch os.Getenv("ZERO_COMMAND_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") + child.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") + cmd.Env = append(os.Environ(), + "ZERO_COMMAND_TREE_HELPER=root", + "ZERO_COMMAND_TREE_PID_FILE="+pidFile, + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + + started := time.Now() + err := RunCommand(ctx, cmd) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) + } + if err == nil { + t.Fatal("timed-out command unexpectedly succeeded") + } + pidData, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + } + pid, parseErr := strconv.Atoi(string(pidData)) + if parseErr != nil { + t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + } + awaitProcessExit(t, pid) +} diff --git a/internal/execution/command_context_unix_test.go b/internal/execution/command_context_unix_test.go new file mode 100644 index 000000000..2be93eb9f --- /dev/null +++ b/internal/execution/command_context_unix_test.go @@ -0,0 +1,19 @@ +//go:build !windows + +package execution + +import ( + "testing" + "time" +) + +func awaitProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for signalTargetRunning(pid) { + if time.Now().After(deadline) { + t.Fatalf("descendant process %d is still running after command cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go new file mode 100644 index 000000000..9ad436051 --- /dev/null +++ b/internal/execution/command_context_windows_test.go @@ -0,0 +1,36 @@ +//go:build windows + +package execution + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const processStillActive = 259 + +func awaitProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for processIsActive(pid) { + if time.Now().After(deadline) { + t.Fatalf("descendant process %d is still running after command cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} + +func processIsActive(pid int) bool { + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return false + } + defer windows.CloseHandle(handle) + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + return false + } + return exitCode == processStillActive +} diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go new file mode 100644 index 000000000..a0c3e45ee --- /dev/null +++ b/internal/execution/command_tree_unix.go @@ -0,0 +1,41 @@ +//go:build !windows + +package execution + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +type commandTree struct { + ready chan struct{} + pid int +} + +func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { + ConfigureProcessGroup(command) + return &commandTree{ready: make(chan struct{})}, nil +} + +func (tree *commandTree) attach(process *os.Process) error { + if process != nil { + tree.pid = process.Pid + } + close(tree.ready) + return nil +} + +func (tree *commandTree) cancel() error { + <-tree.ready + if tree.pid <= 1 { + return nil + } + if err := syscall.Kill(-tree.pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} + +func (tree *commandTree) close() error { return tree.cancel() } diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go new file mode 100644 index 000000000..73fa69fbc --- /dev/null +++ b/internal/execution/command_tree_windows.go @@ -0,0 +1,60 @@ +//go:build windows + +package execution + +import ( + "fmt" + "os" + "os/exec" + "unsafe" + + "golang.org/x/sys/windows" +) + +type commandTree struct { + job windows.Handle + ready chan struct{} +} + +func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("execution: create process job: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + _ = windows.CloseHandle(job) + return nil, fmt.Errorf("execution: configure process job: %w", err) + } + return &commandTree{job: job, ready: make(chan struct{})}, nil +} + +func (tree *commandTree) attach(process *os.Process) error { + defer close(tree.ready) + if process == nil { + return nil + } + handle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(process.Pid), + ) + if err != nil { + return err + } + defer windows.CloseHandle(handle) + return windows.AssignProcessToJobObject(tree.job, handle) +} + +func (tree *commandTree) cancel() error { + <-tree.ready + return windows.TerminateJobObject(tree.job, 1) +} + +func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index 946ed4b3a..3b5c998ab 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -298,7 +298,6 @@ func (dispatcher *Dispatcher) recordCompleted(hook Definition, input DispatchInp // ExitCode (not Err); Err is reserved for commands that could not be launched. func execCommandRunner(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { cmd := exec.CommandContext(ctx, command, args...) - execution.HardenCommandContext(cmd) cmd.Dir = cwd cmd.Env = env if len(stdin) > 0 { @@ -307,7 +306,7 @@ func execCommandRunner(ctx context.Context, command string, args []string, stdin var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result := commandResult{Stdout: stdout.String(), Stderr: stderr.String()} if err == nil { return result diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 099f3b5be..c99e70e2f 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "testing" "time" @@ -36,29 +37,52 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { if err := child.Start(); err != nil { os.Exit(2) } + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } select {} case "grandchild": time.Sleep(30 * time.Second) os.Exit(0) } + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() started := time.Now() - result := execCommandRunner( - ctx, - os.Args[0], - []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, - nil, - "", - append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent"), - ) + resultChannel := make(chan commandResult, 1) + go func() { + resultChannel <- execCommandRunner( + ctx, + os.Args[0], + []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, + nil, + "", + append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent", "ZERO_HOOK_TREE_PID_FILE="+pidFile), + ) + }() + var result commandResult + select { + case result = <-resultChannel: + case <-time.After(4 * time.Second): + cancel() + t.Fatal("execCommandRunner did not return within four seconds after its timeout") + } if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) } if result.Err == nil && result.ExitCode == 0 { t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) } + pidData, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read grandchild PID: %v", err) + } + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("parse grandchild PID %q: %v", pidData, err) + } + awaitHookProcessExit(t, pid) } func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { diff --git a/internal/hooks/process_test_unix.go b/internal/hooks/process_test_unix.go new file mode 100644 index 000000000..01e22f2a8 --- /dev/null +++ b/internal/hooks/process_test_unix.go @@ -0,0 +1,28 @@ +//go:build !windows + +package hooks + +import ( + "errors" + "syscall" + "testing" + "time" +) + +func awaitHookProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + return + } + if err != nil { + t.Fatalf("probe grandchild process %d: %v", pid, err) + } + if time.Now().After(deadline) { + t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/hooks/process_test_windows.go b/internal/hooks/process_test_windows.go new file mode 100644 index 000000000..2b859f545 --- /dev/null +++ b/internal/hooks/process_test_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package hooks + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const processStillActive = 259 + +func awaitHookProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for hookProcessIsActive(pid) { + if time.Now().After(deadline) { + t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} + +func hookProcessIsActive(pid int) bool { + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return false + } + defer windows.CloseHandle(handle) + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + return false + } + return exitCode == processStillActive +} diff --git a/internal/perfbench/perfbench.go b/internal/perfbench/perfbench.go index 762ee1ee9..97435ff83 100644 --- a/internal/perfbench/perfbench.go +++ b/internal/perfbench/perfbench.go @@ -267,12 +267,14 @@ func MeasureColdStart(ctx context.Context, command []string) (float64, error) { } startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) - output, err := cmd.CombinedOutput() + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + err := execution.RunCommand(ctx, cmd) durationMs := RoundMetric(float64(time.Since(startedAt).Microseconds()) / 1000) if err != nil { - return 0, commandError(command, err, string(output), "") + return 0, commandError(command, err, output.String(), "") } return durationMs, nil } @@ -284,7 +286,6 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl rssBefore := readHarnessMemoryMb() startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Env = offlineBenchmarkEnv(os.Environ()) var once sync.Once @@ -298,7 +299,7 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl stderr := &timedBuffer{onFirstWrite: markFirstOutput} cmd.Stdout = stdout cmd.Stderr = stderr - waitErr := cmd.Run() + waitErr := execution.RunCommand(ctx, cmd) finishedAt := time.Now() if firstOutputAt.IsZero() { diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index ea9a0d86f..a922bd998 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -330,7 +330,6 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { return func(ctx context.Context, task BenchTask, rc RunContext) TaskOutcome { args := buildExecArgs(task, rc, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -338,7 +337,10 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - runErr := cmd.Run() + runErr := execution.RunCommand(ctx, cmd) + if errors.Is(runErr, exec.ErrWaitDelay) { + return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + } // The terminal run_end exit code is authoritative for pass/fail: a non-zero // agent exit is a normal task failure, not a harness error, even though @@ -386,7 +388,6 @@ func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { func runVerification(ctx context.Context, task BenchTask) TaskOutcome { cmd := exec.CommandContext(ctx, task.VerificationCommand[0], task.VerificationCommand[1:]...) - execution.HardenCommandContext(cmd) // Inherit the environment so the verifier sees PATH, HOME, language toolchain // vars, etc. — the same surface a maintainer gets running the command by hand // (matching the agent run above). NO_COLOR is appended for stable output. @@ -394,9 +395,12 @@ func runVerification(ctx context.Context, task BenchTask) TaskOutcome { if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir } - output, err := cmd.CombinedOutput() + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + err := execution.RunCommand(ctx, cmd) if err != nil { - detail := strings.TrimSpace(string(output)) + detail := strings.TrimSpace(output.String()) if detail == "" { detail = err.Error() } diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index 4f08d072f..ee55367b5 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -319,6 +319,21 @@ exit 0 } } +func TestNewExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { + stub := writeExecStub(t, `sleep 30 & +echo '{"type":"run_end","exitCode":0}' +exit 0 +`) + runner := NewExecRunner(stub) + outcome := runner(context.Background(), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + if outcome.Err == nil || !strings.Contains(outcome.Err.Error(), "output cleanup failed") { + t.Fatalf("inherited output pipe must be a harness error, got %#v", outcome) + } + if outcome.Passed { + t.Fatal("run_end must not bypass an output cleanup failure") + } +} + func TestNewExecRunnerLaunchFailureIsHarnessError(t *testing.T) { // A binary that cannot be launched (no terminal event, process error) is a // genuine harness error. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 788a95944..1e30db2cf 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -658,7 +658,6 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { args := buildTurnExecArgs(task, rc, tracePath, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -667,12 +666,15 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { cmd.Stdout = &outBuf cmd.Stderr = &errBuf start := time.Now() - runErr := cmd.Run() + runErr := execution.RunCommand(ctx, cmd) wallMs := float64(time.Since(start).Microseconds()) / 1000 exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) outcome := TurnTaskOutcome{WallMs: wallMs} - if haveExit && exitCode != 0 { + if errors.Is(runErr, exec.ErrWaitDelay) { + outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + return outcome + } else if haveExit && exitCode != 0 { outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode) } else if !haveExit { detail := strings.TrimSpace(errBuf.String()) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 5b2295654..d91597241 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -643,6 +643,20 @@ func runTurnStub(t *testing.T, task BenchTask, stubBody string) TurnTaskOutcome return NewTurnExecRunner(stub)(context.Background(), task, RunContext{Model: "fake-model"}) } +func TestNewTurnExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { + task := BenchTask{ID: "wait-delay", Prompt: "p", WorkspaceFixture: t.TempDir()} + outcome := runTurnStub(t, task, `sleep 30 & +echo '{"type":"run_end","exitCode":0}' +exit 0 +`) + if outcome.Err == nil || !strings.Contains(outcome.Err.Error(), "output cleanup failed") { + t.Fatalf("inherited output pipe must be a harness error, got %#v", outcome) + } + if outcome.Passed { + t.Fatal("run_end must not bypass an output cleanup failure") + } +} + // assertVerifyFailed asserts an outcome failed specifically because the oracle // rejected the work — Passed is false, there is no harness error (Err nil), and // VerifyErr carries the surfaced failure detail. This is stronger than merely diff --git a/internal/verify/verify.go b/internal/verify/verify.go index cd152bf41..a571cada8 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -3,6 +3,7 @@ package verify import ( "bytes" "context" + "errors" "fmt" "os" "os/exec" @@ -300,17 +301,17 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti defer cancel() cmd := exec.CommandContext(commandCtx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = dir var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(commandCtx, cmd) exitCode := 0 if err != nil { exitCode = -1 - if exitError, ok := err.(*exec.ExitError); ok { + var exitError *exec.ExitError + if errors.As(err, &exitError) { exitCode = exitError.ExitCode() err = nil } diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go index 5a9291614..b8792ca09 100644 --- a/internal/verify/verify_test.go +++ b/internal/verify/verify_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -12,6 +13,41 @@ import ( "github.com/Gitlawb/zero/internal/testrunner" ) +func TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { + switch os.Getenv("ZERO_VERIFY_TREE_HELPER") { + case "parent": + if err := os.Setenv("ZERO_VERIFY_TREE_HELPER", "grandchild"); err != nil { + os.Exit(2) + } + child := exec.Command(os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$") + child.Env = os.Environ() + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(3) + } + select {} + case "grandchild": + time.Sleep(30 * time.Second) + return + } + + t.Setenv("ZERO_VERIFY_TREE_HELPER", "parent") + plan := Plan{Root: t.TempDir(), Checks: []Check{{ + ID: "tree.timeout", + Name: "process tree timeout", + Command: []string{os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$"}, + }}} + started := time.Now() + report := Run(context.Background(), plan, RunOptions{TimeoutMS: 100}) + if elapsed := time.Since(started); elapsed > 2*time.Second { + t.Fatalf("defaultRunner remained blocked by grandchild output handles for %s", elapsed) + } + if report.OK || len(report.Results) != 1 || report.Results[0].Status == StatusPass { + t.Fatalf("timed-out defaultRunner command unexpectedly passed: %#v", report) + } +} + func TestDetectPlanFindsBunAndGoChecks(t *testing.T) { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), "module example.com/zero\n") From f74301817a67c4012d59f24d8ada9c5cc1777a79 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 27 Aug 2026 21:21:39 +0000 Subject: [PATCH 03/17] fix(process): bind Windows jobs before execution Amp-Thread-ID: https://ampcode.com/threads/T-01a044e2-92ad-774b-9a86-094e2e5293bf Co-authored-by: Pierre Bruno --- internal/execution/command_tree_unix.go | 2 +- internal/execution/command_tree_windows.go | 46 ++++++++++++++++++++-- internal/hooks/dispatch_test.go | 2 +- internal/perfbench/taskbench_test.go | 2 +- internal/perfbench/turn_bench_test.go | 2 +- 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go index a0c3e45ee..eb00dfef1 100644 --- a/internal/execution/command_tree_unix.go +++ b/internal/execution/command_tree_unix.go @@ -38,4 +38,4 @@ func (tree *commandTree) cancel() error { return nil } -func (tree *commandTree) close() error { return tree.cancel() } +func (*commandTree) close() error { return nil } diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index 73fa69fbc..d70e36bbc 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,9 +3,11 @@ package execution import ( + "errors" "fmt" "os" "os/exec" + "syscall" "unsafe" "golang.org/x/sys/windows" @@ -16,7 +18,7 @@ type commandTree struct { ready chan struct{} } -func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { +func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { job, err := windows.CreateJobObject(nil, nil) if err != nil { return nil, fmt.Errorf("execution: create process job: %w", err) @@ -32,6 +34,10 @@ func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { _ = windows.CloseHandle(job) return nil, fmt.Errorf("execution: configure process job: %w", err) } + if command.SysProcAttr == nil { + command.SysProcAttr = &syscall.SysProcAttr{} + } + command.SysProcAttr.CreationFlags |= windows.CREATE_SUSPENDED return &commandTree{job: job, ready: make(chan struct{})}, nil } @@ -48,8 +54,13 @@ func (tree *commandTree) attach(process *os.Process) error { if err != nil { return err } - defer windows.CloseHandle(handle) - return windows.AssignProcessToJobObject(tree.job, handle) + if err := windows.AssignProcessToJobObject(tree.job, handle); err != nil { + return errors.Join(err, windows.CloseHandle(handle)) + } + if err := windows.CloseHandle(handle); err != nil { + return err + } + return resumeProcess(uint32(process.Pid)) } func (tree *commandTree) cancel() error { @@ -58,3 +69,32 @@ func (tree *commandTree) cancel() error { } func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } + +func resumeProcess(pid uint32) (err error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return err + } + defer func() { err = errors.Join(err, windows.CloseHandle(snapshot)) }() + + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + if err := windows.Thread32First(snapshot, &entry); err != nil { + return err + } + for { + if entry.OwnerProcessID == pid { + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + if err != nil { + return err + } + _, resumeErr := windows.ResumeThread(thread) + return errors.Join(resumeErr, windows.CloseHandle(thread)) + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + return fmt.Errorf("execution: no thread found for suspended process %d", pid) + } + return err + } + } +} diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index c99e70e2f..4ce1b9ba7 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -47,7 +47,7 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { } pidFile := filepath.Join(t.TempDir(), "grandchild.pid") - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() started := time.Now() resultChannel := make(chan commandResult, 1) diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index ee55367b5..e42d0af82 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -320,7 +320,7 @@ exit 0 } func TestNewExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { - stub := writeExecStub(t, `sleep 30 & + stub := writeExecStub(t, `sleep 3 & echo '{"type":"run_end","exitCode":0}' exit 0 `) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index d91597241..2be6d1a1f 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -645,7 +645,7 @@ func runTurnStub(t *testing.T, task BenchTask, stubBody string) TurnTaskOutcome func TestNewTurnExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { task := BenchTask{ID: "wait-delay", Prompt: "p", WorkspaceFixture: t.TempDir()} - outcome := runTurnStub(t, task, `sleep 30 & + outcome := runTurnStub(t, task, `sleep 3 & echo '{"type":"run_end","exitCode":0}' exit 0 `) From 44aff3d717df09f1afbc4422e7882a9e376224a9 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Fri, 28 Aug 2026 20:57:11 +0200 Subject: [PATCH 04/17] fix(execution): preserve successful Windows descendants --- .../execution/command_context_windows_test.go | 72 +++++++++++++++++++ internal/execution/command_tree_windows.go | 11 --- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go index 9ad436051..bae0f7577 100644 --- a/internal/execution/command_context_windows_test.go +++ b/internal/execution/command_context_windows_test.go @@ -3,6 +3,12 @@ package execution import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" "testing" "time" @@ -34,3 +40,69 @@ func processIsActive(pid int) bool { } return exitCode == processStillActive } + +func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { + switch os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) + } + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") + child.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child") + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + child.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: windows.DETACHED_PROCESS | windows.CREATE_NEW_PROCESS_GROUP, + } + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(4) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx := context.Background() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") + command.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=root", + "ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE="+pidFile, + ) + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand failed: %v", err) + } + pidData, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read detached child PID: %v", err) + } + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("parse detached child PID %q: %v", pidData, err) + } + t.Cleanup(func() { + if !processIsActive(pid) { + return + } + process, findErr := os.FindProcess(pid) + if findErr != nil { + t.Errorf("find detached child %d: %v", pid, findErr) + return + } + if killErr := process.Kill(); killErr != nil { + t.Errorf("kill detached child %d: %v", pid, killErr) + return + } + awaitProcessExit(t, pid) + }) + if !processIsActive(pid) { + t.Fatalf("successful RunCommand terminated detached child %d", pid) + } +} diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index d70e36bbc..4fb7ea9d8 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -23,17 +23,6 @@ func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { if err != nil { return nil, fmt.Errorf("execution: create process job: %w", err) } - info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} - info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - if _, err := windows.SetInformationJobObject( - job, - windows.JobObjectExtendedLimitInformation, - uintptr(unsafe.Pointer(&info)), - uint32(unsafe.Sizeof(info)), - ); err != nil { - _ = windows.CloseHandle(job) - return nil, fmt.Errorf("execution: configure process job: %w", err) - } if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } From e9af6888ecdb2500bf9e0d8e203897f5f33611aa Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sat, 29 Aug 2026 10:38:11 +0200 Subject: [PATCH 05/17] fix(execution): fall back when tree containment fails Amp-Thread-ID: https://ampcode.com/threads/T-01a04c92-2d1d-7508-91bc-416341b7e8b0 Co-authored-by: Amp --- internal/execution/command_context.go | 3 + internal/execution/command_context_test.go | 50 ++++++++++ internal/execution/command_tree_windows.go | 93 +++++++++++++++---- .../execution/command_tree_windows_test.go | 77 +++++++++++++++ 4 files changed, 206 insertions(+), 17 deletions(-) create mode 100644 internal/execution/command_tree_windows_test.go diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 6e40af31a..015a7e151 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -53,5 +53,8 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if canceled.canceled { return errors.Join(waitErr, ctx.Err(), canceled.err) } + if waitErr != nil { + return errors.Join(waitErr, tree.cancel()) + } return waitErr } diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index 7c0c052da..2af3642ff 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -3,6 +3,7 @@ package execution import ( "bytes" "context" + "errors" "os" "os/exec" "strconv" @@ -59,3 +60,52 @@ func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { } awaitProcessExit(t, pid) } + +func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { + switch os.Getenv("ZERO_WAIT_DELAY_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") + child.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" + ctx := context.Background() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") + cmd.Env = append(os.Environ(), + "ZERO_WAIT_DELAY_TREE_HELPER=root", + "ZERO_WAIT_DELAY_TREE_PID_FILE="+pidFile, + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + + started := time.Now() + err := RunCommand(ctx, cmd) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) + } + if !errors.Is(err, exec.ErrWaitDelay) { + t.Fatalf("RunCommand error = %v, want exec.ErrWaitDelay", err) + } + pidData, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + } + pid, parseErr := strconv.Atoi(string(pidData)) + if parseErr != nil { + t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + } + awaitProcessExit(t, pid) +} diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index 4fb7ea9d8..dcf2bce2a 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,10 +3,12 @@ package execution import ( + "context" "errors" "fmt" "os" "os/exec" + "strconv" "syscall" "unsafe" @@ -14,15 +16,20 @@ import ( ) type commandTree struct { - job windows.Handle - ready chan struct{} + job windows.Handle + processHandle windows.Handle + process *os.Process + contained bool + ready chan struct{} } +const commandProcessStillActive = uint32(259) // STILL_ACTIVE + func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - job, err := windows.CreateJobObject(nil, nil) - if err != nil { - return nil, fmt.Errorf("execution: create process job: %w", err) - } + // Job containment is preferred but optional. The process still starts + // suspended so attach can retain its identity and either assign the job or + // establish the fallback before any child process can escape. + job, _ := windows.CreateJobObject(nil, nil) if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } @@ -35,29 +42,81 @@ func (tree *commandTree) attach(process *os.Process) error { if process == nil { return nil } + tree.process = process handle, err := windows.OpenProcess( - windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(process.Pid), ) - if err != nil { - return err - } - if err := windows.AssignProcessToJobObject(tree.job, handle); err != nil { - return errors.Join(err, windows.CloseHandle(handle)) - } - if err := windows.CloseHandle(handle); err != nil { - return err + if err == nil { + tree.processHandle = handle + if tree.job != 0 && assignCommandProcessToJob(tree.job, handle) == nil { + tree.contained = true + } + } else { + // PROCESS_SET_QUOTA is needed only for job assignment. If the host + // denies that setup right, retry with the narrower rights needed by the + // retained identity-safe fallback. + tree.processHandle, _ = windows.OpenProcess( + windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, + false, + uint32(process.Pid), + ) } + // Job creation, process opening, and assignment can all fail when the host + // already constrains this process. They must not strand it suspended. return resumeProcess(uint32(process.Pid)) } func (tree *commandTree) cancel() error { <-tree.ready - return windows.TerminateJobObject(tree.job, 1) + if tree.contained { + return windows.TerminateJobObject(tree.job, 1) + } + if tree.processHandle == 0 { + if tree.process == nil { + return nil + } + return tree.process.Kill() + } + + // The retained handle both identifies the original process and prevents its + // PID from being reused. Only ask taskkill to walk by PID while that exact + // process is still active; after Wait has reaped it, numeric PID targeting + // could otherwise hit an unrelated process. + var exitCode uint32 + if err := windows.GetExitCodeProcess(tree.processHandle, &exitCode); err != nil { + return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) + } + if exitCode != commandProcessStillActive { + return nil + } + if err := cancelCommandTreeByPID(tree.process.Pid); err == nil { + return nil + } else { + return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) + } } -func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } +func (tree *commandTree) close() (err error) { + if tree.job != 0 { + err = errors.Join(err, windows.CloseHandle(tree.job)) + tree.job = 0 + } + if tree.processHandle != 0 { + err = errors.Join(err, windows.CloseHandle(tree.processHandle)) + tree.processHandle = 0 + } + return err +} + +var assignCommandProcessToJob = windows.AssignProcessToJobObject + +var cancelCommandTreeByPID = func(pid int) error { + ctx, cancel := context.WithTimeout(context.Background(), processWaitDelay) + defer cancel() + return exec.CommandContext(ctx, taskkillPath(), "/T", "/F", "/PID", strconv.Itoa(pid)).Run() +} func resumeProcess(pid uint32) (err error) { snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go new file mode 100644 index 000000000..ef9ac9de0 --- /dev/null +++ b/internal/execution/command_tree_windows_test.go @@ -0,0 +1,77 @@ +//go:build windows + +package execution + +import ( + "context" + "os/exec" + "testing" + + "golang.org/x/sys/windows" +) + +func TestRunCommandContinuesWhenJobAssignmentFails(t *testing.T) { + originalAssign := assignCommandProcessToJob + assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + return windows.ERROR_ACCESS_DENIED + } + t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) + + ctx := context.Background() + command := exec.CommandContext(ctx, "cmd", "/C", "exit /b 0") + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand failed after optional job assignment failed: %v", err) + } +} + +func TestCommandTreeFallbackDoesNotTargetExitedProcessPID(t *testing.T) { + originalAssign := assignCommandProcessToJob + assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + return windows.ERROR_ACCESS_DENIED + } + t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) + + taskkillCalls := 0 + originalCancelByPID := cancelCommandTreeByPID + cancelCommandTreeByPID = func(int) error { + taskkillCalls++ + return nil + } + t.Cleanup(func() { cancelCommandTreeByPID = originalCancelByPID }) + + command := exec.Command("cmd", "/C", "exit /b 0") + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepareCommandTree: %v", err) + } + t.Cleanup(func() { + if closeErr := tree.close(); closeErr != nil { + t.Errorf("close command tree: %v", closeErr) + } + }) + if err := command.Start(); err != nil { + _ = tree.attach(nil) + t.Fatalf("Start: %v", err) + } + if err := tree.attach(command.Process); err != nil { + _ = command.Process.Kill() + _ = command.Wait() + t.Fatalf("attachCommandTree: %v", err) + } + if tree.contained { + t.Fatal("command unexpectedly reported job containment after forced assignment failure") + } + if tree.processHandle == 0 { + t.Fatal("command tree did not retain the fallback process identity") + } + if err := command.Wait(); err != nil { + t.Fatalf("Wait: %v", err) + } + + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after root exit: %v", err) + } + if taskkillCalls != 0 { + t.Fatalf("fallback targeted an exited root's numeric PID %d time(s)", taskkillCalls) + } +} From 91c5895c62d8902af7a65ae7543c5313f0e495f8 Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 2 Sep 2026 22:04:33 +0000 Subject: [PATCH 06/17] fix(execution): require Windows job containment Amp-Thread-ID: https://ampcode.com/threads/T-01a06415-151c-75f8-9316-7cf572d58d9b Co-authored-by: Pierre Bruno --- internal/execution/command_tree_windows.go | 70 +++----------- .../execution/command_tree_windows_test.go | 91 ++++++++----------- 2 files changed, 54 insertions(+), 107 deletions(-) diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index dcf2bce2a..db5a64ab3 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,12 +3,10 @@ package execution import ( - "context" "errors" "fmt" "os" "os/exec" - "strconv" "syscall" "unsafe" @@ -18,18 +16,15 @@ import ( type commandTree struct { job windows.Handle processHandle windows.Handle - process *os.Process contained bool ready chan struct{} } -const commandProcessStillActive = uint32(259) // STILL_ACTIVE - func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - // Job containment is preferred but optional. The process still starts - // suspended so attach can retain its identity and either assign the job or - // establish the fallback before any child process can escape. - job, _ := windows.CreateJobObject(nil, nil) + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("execution: create command job: %w", err) + } if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } @@ -42,29 +37,22 @@ func (tree *commandTree) attach(process *os.Process) error { if process == nil { return nil } - tree.process = process handle, err := windows.OpenProcess( windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(process.Pid), ) - if err == nil { - tree.processHandle = handle - if tree.job != 0 && assignCommandProcessToJob(tree.job, handle) == nil { - tree.contained = true - } - } else { - // PROCESS_SET_QUOTA is needed only for job assignment. If the host - // denies that setup right, retry with the narrower rights needed by the - // retained identity-safe fallback. - tree.processHandle, _ = windows.OpenProcess( - windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, - false, - uint32(process.Pid), - ) + if err != nil { + return fmt.Errorf("open suspended command process: %w", err) + } + tree.processHandle = handle + if err := assignCommandProcessToJob(tree.job, handle); err != nil { + // Without job containment, a root can exit before cancellation and + // leave no identity-safe way to find descendants holding output pipes. + // Fail while it is still suspended so no descendant can escape. + return fmt.Errorf("assign suspended command process to job: %w", err) } - // Job creation, process opening, and assignment can all fail when the host - // already constrains this process. They must not strand it suspended. + tree.contained = true return resumeProcess(uint32(process.Pid)) } @@ -73,29 +61,7 @@ func (tree *commandTree) cancel() error { if tree.contained { return windows.TerminateJobObject(tree.job, 1) } - if tree.processHandle == 0 { - if tree.process == nil { - return nil - } - return tree.process.Kill() - } - - // The retained handle both identifies the original process and prevents its - // PID from being reused. Only ask taskkill to walk by PID while that exact - // process is still active; after Wait has reaped it, numeric PID targeting - // could otherwise hit an unrelated process. - var exitCode uint32 - if err := windows.GetExitCodeProcess(tree.processHandle, &exitCode); err != nil { - return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) - } - if exitCode != commandProcessStillActive { - return nil - } - if err := cancelCommandTreeByPID(tree.process.Pid); err == nil { - return nil - } else { - return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) - } + return nil } func (tree *commandTree) close() (err error) { @@ -112,12 +78,6 @@ func (tree *commandTree) close() (err error) { var assignCommandProcessToJob = windows.AssignProcessToJobObject -var cancelCommandTreeByPID = func(pid int) error { - ctx, cancel := context.WithTimeout(context.Background(), processWaitDelay) - defer cancel() - return exec.CommandContext(ctx, taskkillPath(), "/T", "/F", "/PID", strconv.Itoa(pid)).Run() -} - func resumeProcess(pid uint32) (err error) { snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) if err != nil { diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go index ef9ac9de0..06caa260c 100644 --- a/internal/execution/command_tree_windows_test.go +++ b/internal/execution/command_tree_windows_test.go @@ -4,74 +4,61 @@ package execution import ( "context" + "errors" + "os" "os/exec" + "path/filepath" + "strconv" "testing" + "time" "golang.org/x/sys/windows" ) -func TestRunCommandContinuesWhenJobAssignmentFails(t *testing.T) { - originalAssign := assignCommandProcessToJob - assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { - return windows.ERROR_ACCESS_DENIED - } - t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - - ctx := context.Background() - command := exec.CommandContext(ctx, "cmd", "/C", "exit /b 0") - if err := RunCommand(ctx, command); err != nil { - t.Fatalf("RunCommand failed after optional job assignment failed: %v", err) +func TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails(t *testing.T) { + switch os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") + child.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return } -} -func TestCommandTreeFallbackDoesNotTargetExitedProcessPID(t *testing.T) { originalAssign := assignCommandProcessToJob assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { return windows.ERROR_ACCESS_DENIED } t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - taskkillCalls := 0 - originalCancelByPID := cancelCommandTreeByPID - cancelCommandTreeByPID = func(int) error { - taskkillCalls++ - return nil - } - t.Cleanup(func() { cancelCommandTreeByPID = originalCancelByPID }) - - command := exec.Command("cmd", "/C", "exit /b 0") - tree, err := prepareCommandTree(command) - if err != nil { - t.Fatalf("prepareCommandTree: %v", err) - } - t.Cleanup(func() { - if closeErr := tree.close(); closeErr != nil { - t.Errorf("close command tree: %v", closeErr) - } - }) - if err := command.Start(); err != nil { - _ = tree.attach(nil) - t.Fatalf("Start: %v", err) - } - if err := tree.attach(command.Process); err != nil { - _ = command.Process.Kill() - _ = command.Wait() - t.Fatalf("attachCommandTree: %v", err) - } - if tree.contained { - t.Fatal("command unexpectedly reported job containment after forced assignment failure") - } - if tree.processHandle == 0 { - t.Fatal("command tree did not retain the fallback process identity") + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx := context.Background() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") + command.Env = append(os.Environ(), + "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=root", + "ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE="+pidFile, + ) + started := time.Now() + err := RunCommand(ctx, command) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand took %s after job assignment failed", elapsed) } - if err := command.Wait(); err != nil { - t.Fatalf("Wait: %v", err) + if !errors.Is(err, windows.ERROR_ACCESS_DENIED) { + t.Fatalf("RunCommand error = %v, want ERROR_ACCESS_DENIED", err) } - - if err := tree.cancel(); err != nil { - t.Fatalf("cancel after root exit: %v", err) + if command.ProcessState == nil || !command.ProcessState.Exited() { + t.Fatalf("suspended command was not killed and reaped: state = %v", command.ProcessState) } - if taskkillCalls != 0 { - t.Fatalf("fallback targeted an exited root's numeric PID %d time(s)", taskkillCalls) + if _, statErr := os.Stat(pidFile); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("suspended command spawned a descendant after job assignment failed: PID file error = %v", statErr) } } From 803e076d5c282237f7ab76be9caffe5decd24e7b Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Thu, 3 Sep 2026 21:20:29 +0200 Subject: [PATCH 07/17] fix(execution): retain process tree lifecycle ownership --- internal/agenteval/agent_command.go | 4 +- internal/agenteval/run.go | 3 +- .../command_context_process_unix_test.go | 100 ++++++++++ .../command_context_process_windows_test.go | 174 ++++++++++++++++++ internal/execution/command_context_test.go | 155 ++++++++++++---- .../execution/command_context_unix_test.go | 60 +++++- .../execution/command_context_windows_test.go | 71 ++----- internal/execution/command_tree_unix.go | 79 ++++++-- internal/execution/command_tree_unix_test.go | 174 ++++++++++++++++++ .../execution/command_tree_windows_test.go | 36 ++-- internal/execution/exit_error.go | 31 ++++ internal/execution/exit_error_test.go | 41 +++++ internal/execution/runner.go | 3 +- internal/hooks/dispatch.go | 4 +- internal/hooks/dispatch_test.go | 100 ++++++++-- internal/hooks/process_test_unix.go | 28 --- internal/hooks/process_test_windows.go | 36 ---- internal/hooks/process_unix_test.go | 81 ++++++++ internal/hooks/process_windows_test.go | 95 ++++++++++ internal/perfbench/taskbench.go | 18 +- internal/perfbench/taskbench_test.go | 109 +++++++++++ internal/perfbench/turn_bench.go | 14 +- internal/perfbench/turn_bench_test.go | 53 +++++- internal/verify/process_unix_test.go | 81 ++++++++ internal/verify/process_windows_test.go | 95 ++++++++++ internal/verify/verify.go | 4 +- internal/verify/verify_test.go | 94 +++++++++- 27 files changed, 1512 insertions(+), 231 deletions(-) create mode 100644 internal/execution/command_context_process_unix_test.go create mode 100644 internal/execution/command_context_process_windows_test.go create mode 100644 internal/execution/command_tree_unix_test.go create mode 100644 internal/execution/exit_error.go create mode 100644 internal/execution/exit_error_test.go delete mode 100644 internal/hooks/process_test_unix.go delete mode 100644 internal/hooks/process_test_windows.go create mode 100644 internal/hooks/process_unix_test.go create mode 100644 internal/hooks/process_windows_test.go create mode 100644 internal/verify/process_unix_test.go create mode 100644 internal/verify/process_windows_test.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index 8919529ea..3c2a3f59e 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -3,7 +3,6 @@ package agenteval import ( "bytes" "context" - "errors" "os/exec" "strings" @@ -83,8 +82,7 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A result.Error = ctxErr.Error() return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index 07a60471a..4e27cffaf 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -153,8 +153,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command result.ExitCode = 0 return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/execution/command_context_process_unix_test.go b/internal/execution/command_context_process_unix_test.go new file mode 100644 index 000000000..3e23eb4d0 --- /dev/null +++ b/internal/execution/command_context_process_unix_test.go @@ -0,0 +1,100 @@ +//go:build !windows + +package execution + +import ( + "errors" + "os" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +type helperProcessOwner struct { + pidFile string + stopFile string + pid int + exited bool +} + +func ownHelperProcess(t *testing.T, pidFile, stopFile string) *helperProcessOwner { + t.Helper() + owner := &helperProcessOwner{pidFile: pidFile, stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperProcessOwner) waitReady(t *testing.T, timeout time.Duration) int { + t.Helper() + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(owner.pidFile) + if err == nil { + pid, parseErr := strconv.Atoi(strings.TrimSpace(string(data))) + if parseErr == nil && pid > 0 { + owner.pid = pid + return pid + } + } + if time.Now().After(deadline) { + t.Fatalf("helper did not hand off a valid PID within %s", timeout) + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *helperProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request helper process stop: %v", err) + } + if owner.exited { + return + } + if owner.pid == 0 { + data, err := os.ReadFile(owner.pidFile) + if err == nil { + owner.pid, _ = strconv.Atoi(strings.TrimSpace(string(data))) + } + } + if owner.pid <= 0 { + return + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + err := syscall.Kill(owner.pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + owner.pid = 0 + owner.exited = true + return + } + if err != nil { + t.Errorf("check helper process %d after cleanup: %v", owner.pid, err) + return + } + time.Sleep(10 * time.Millisecond) + } + t.Errorf("helper process %d survived cleanup", owner.pid) +} + +func (owner *helperProcessOwner) awaitExit(t *testing.T) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(owner.pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + owner.pid = 0 + owner.exited = true + return + } + if err != nil { + t.Fatalf("check helper process %d: %v", owner.pid, err) + } + if time.Now().After(deadline) { + t.Fatalf("helper process %d is still running after command cancellation", owner.pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/execution/command_context_process_windows_test.go b/internal/execution/command_context_process_windows_test.go new file mode 100644 index 000000000..9364062fb --- /dev/null +++ b/internal/execution/command_context_process_windows_test.go @@ -0,0 +1,174 @@ +//go:build windows + +package execution + +import ( + "errors" + "os" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type helperProcessOwner struct { + pidFile string + stopFile string + pid int + handle windows.Handle + exited bool +} + +func ownHelperProcess(t *testing.T, pidFile, stopFile string) *helperProcessOwner { + t.Helper() + owner := &helperProcessOwner{pidFile: pidFile, stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperProcessOwner) waitReady(t *testing.T, timeout time.Duration) int { + t.Helper() + deadline := time.Now().Add(timeout) + for { + observed, err := owner.observeReady() + if err != nil { + t.Fatalf("retain helper process: %v", err) + } + if observed { + return owner.pid + } + if time.Now().After(deadline) { + t.Fatalf("helper did not hand off a valid PID within %s", timeout) + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *helperProcessOwner) observeReady() (bool, error) { + data, err := os.ReadFile(owner.pidFile) + if err != nil { + return false, nil + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return false, nil + } + if owner.handle != 0 { + return true, nil + } + if err := owner.retainPID(pid); err != nil { + return true, err + } + return true, nil +} + +func (owner *helperProcessOwner) retainPID(pid int) error { + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + owner.pid = pid + owner.exited = true + return nil + } + if err != nil { + return err + } + owner.pid = pid + owner.handle = handle + return nil +} + +func (owner *helperProcessOwner) running() bool { + if owner.handle == 0 { + return false + } + var exitCode uint32 + return windows.GetExitCodeProcess(owner.handle, &exitCode) == nil && exitCode == processStillActive +} + +func (owner *helperProcessOwner) awaitExit(t *testing.T) { + t.Helper() + if owner.exited { + return + } + if owner.handle == 0 { + t.Fatal("helper process handle was not retained") + } + status, err := windows.WaitForSingleObject(owner.handle, 2_000) + if err != nil { + t.Fatalf("wait for helper process %d: %v", owner.pid, err) + } + if status != windows.WAIT_OBJECT_0 { + t.Fatalf("helper process %d is still running after command cancellation", owner.pid) + } +} + +func (owner *helperProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request helper process stop: %v", err) + } + if owner.handle == 0 { + observed, err := owner.observeReady() + if observed && err != nil { + t.Errorf("retain helper process for cleanup: %v", err) + } + } + if owner.handle == 0 { + return + } + if owner.running() { + status, err := windows.WaitForSingleObject(owner.handle, 2_000) + if err != nil { + t.Errorf("wait for helper process %d cooperative stop: %v", owner.pid, err) + } else if status == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(owner.handle, 1); err != nil { + t.Errorf("kill helper process %d: %v", owner.pid, err) + } + _, _ = windows.WaitForSingleObject(owner.handle, 2_000) + } + } + if err := windows.CloseHandle(owner.handle); err != nil { + t.Errorf("close helper process %d handle: %v", owner.pid, err) + } + owner.handle = 0 +} + +type helperHandleOwner struct { + handle windows.Handle + stopFile string +} + +func ownHelperHandle(t *testing.T, stopFile string) *helperHandleOwner { + t.Helper() + owner := &helperHandleOwner{stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperHandleOwner) retain(process windows.Handle) error { + current := windows.CurrentProcess() + return windows.DuplicateHandle(current, process, current, &owner.handle, windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, 0) +} + +func (owner *helperHandleOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request suspended helper process stop: %v", err) + } + if owner.handle == 0 { + return + } + var exitCode uint32 + if windows.GetExitCodeProcess(owner.handle, &exitCode) == nil && exitCode == processStillActive { + if err := windows.TerminateProcess(owner.handle, 1); err != nil { + t.Errorf("kill suspended helper process: %v", err) + } + _, _ = windows.WaitForSingleObject(owner.handle, 2_000) + } + if err := windows.CloseHandle(owner.handle); err != nil { + t.Errorf("close suspended helper process handle: %v", err) + } + owner.handle = 0 +} diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index 2af3642ff..73cf8fd69 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -15,97 +15,182 @@ func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { switch os.Getenv("ZERO_COMMAND_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") - child.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_COMMAND_TREE_HELPER=child", + "ZERO_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") cmd.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=root", "ZERO_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_COMMAND_TREE_STOP_FILE="+stopFile, ) var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - started := time.Now() - err := RunCommand(ctx, cmd) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) - } + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + cancel() + err := waitForRunCommand(t, result, 4*time.Second) if err == nil { t.Fatal("timed-out command unexpectedly succeeded") } - pidData, readErr := os.ReadFile(pidFile) - if readErr != nil { - t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) - } - pid, parseErr := strconv.Atoi(string(pidData)) - if parseErr != nil { - t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) - } - awaitProcessExit(t, pid) + child.awaitExit(t) } func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { switch os.Getenv("ZERO_WAIT_DELAY_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") - child.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_WAIT_DELAY_TREE_HELPER=child", + "ZERO_WAIT_DELAY_TREE_STOP_FILE="+os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" - ctx := context.Background() + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") cmd.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=root", "ZERO_WAIT_DELAY_TREE_PID_FILE="+pidFile, + "ZERO_WAIT_DELAY_TREE_STOP_FILE="+stopFile, ) var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - started := time.Now() - err := RunCommand(ctx, cmd) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) - } + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + err := waitForRunCommand(t, result, 4*time.Second) if !errors.Is(err, exec.ErrWaitDelay) { t.Fatalf("RunCommand error = %v, want exec.ErrWaitDelay", err) } - pidData, readErr := os.ReadFile(pidFile) - if readErr != nil { - t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + child.awaitExit(t) +} + +func TestRunCommandKillsDescendantAfterNonzeroRootExit(t *testing.T) { + switch os.Getenv("ZERO_NONZERO_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) + } + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterNonzeroRootExit$") + child.Env = append(os.Environ(), + "ZERO_NONZERO_TREE_HELPER=child", + "ZERO_NONZERO_TREE_STOP_FILE="+os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), + ) + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_NONZERO_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + // Leave time for the parent test to retain an independent cleanup handle + // before the root's abnormal exit triggers production tree cleanup. + if waitForCommandTreeStop(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), 500*time.Millisecond) { + _ = child.Wait() + return + } + os.Exit(7) + case "child": + waitForCommandTreeStop(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), 30*time.Second) + return + } + + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx := context.Background() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterNonzeroRootExit$") + cmd.Env = append(os.Environ(), + "ZERO_NONZERO_TREE_HELPER=root", + "ZERO_NONZERO_TREE_PID_FILE="+pidFile, + "ZERO_NONZERO_TREE_STOP_FILE="+stopFile, + ) + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + err := waitForRunCommand(t, result, 4*time.Second) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 7 { + t.Fatalf("RunCommand error = %v, want exit code 7", err) } - pid, parseErr := strconv.Atoi(string(pidData)) - if parseErr != nil { - t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + child.awaitExit(t) +} + +func waitForCommandTreeStop(stopFile string, lifetime time.Duration) bool { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + +func runCommandAsync(ctx context.Context, command *exec.Cmd) <-chan error { + result := make(chan error, 1) + go func() { result <- RunCommand(ctx, command) }() + return result +} + +func waitForRunCommand(t *testing.T, result <-chan error, timeout time.Duration) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(timeout): + t.Fatalf("RunCommand did not return within %s", timeout) + return nil } - awaitProcessExit(t, pid) } diff --git a/internal/execution/command_context_unix_test.go b/internal/execution/command_context_unix_test.go index 2be93eb9f..f74cb5e5c 100644 --- a/internal/execution/command_context_unix_test.go +++ b/internal/execution/command_context_unix_test.go @@ -3,17 +3,63 @@ package execution import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" "testing" "time" ) -func awaitProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for signalTargetRunning(pid) { - if time.Now().After(deadline) { - t.Fatalf("descendant process %d is still running after command cancellation", pid) +func TestRunCommandPreservesRedirectedChildAfterSuccessfulExit(t *testing.T) { + switch os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) } - time.Sleep(10 * time.Millisecond) + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesRedirectedChildAfterSuccessfulExit$") + child.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER=child", + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), + ) + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + return + case "child": + waitForCommandTreeStop(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), 30*time.Second) + return + } + + root := t.TempDir() + pidFile := filepath.Join(root, "child.pid") + stopFile := filepath.Join(root, "stop") + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesRedirectedChildAfterSuccessfulExit$") + command.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER=root", + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE="+stopFile, + ) + result := runCommandAsync(ctx, command) + pid := child.waitReady(t, 2*time.Second) + if err := waitForRunCommand(t, result, 4*time.Second); err != nil { + t.Fatalf("RunCommand failed: %v", err) + } + if !signalTargetRunning(pid) { + t.Fatalf("successful RunCommand terminated redirected child %d", pid) } } diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go index bae0f7577..fbe862039 100644 --- a/internal/execution/command_context_windows_test.go +++ b/internal/execution/command_context_windows_test.go @@ -17,30 +17,6 @@ import ( const processStillActive = 259 -func awaitProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for processIsActive(pid) { - if time.Now().After(deadline) { - t.Fatalf("descendant process %d is still running after command cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} - -func processIsActive(pid int) bool { - handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) - if err != nil { - return false - } - defer windows.CloseHandle(handle) - var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { - return false - } - return exitCode == processStillActive -} - func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { switch os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_HELPER") { case "root": @@ -50,7 +26,10 @@ func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { } defer nullFile.Close() child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") - child.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child", + "ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), + ) child.Stdin = nullFile child.Stdout = nullFile child.Stderr = nullFile @@ -61,48 +40,34 @@ func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { os.Exit(3) } if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(4) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := filepath.Join(t.TempDir(), "child.pid") - ctx := context.Background() + root := t.TempDir() + pidFile := filepath.Join(root, "child.pid") + stopFile := filepath.Join(root, "stop") + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") command.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=root", "ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE="+stopFile, ) - if err := RunCommand(ctx, command); err != nil { + result := runCommandAsync(ctx, command) + pid := child.waitReady(t, 2*time.Second) + if err := waitForRunCommand(t, result, 4*time.Second); err != nil { t.Fatalf("RunCommand failed: %v", err) } - pidData, err := os.ReadFile(pidFile) - if err != nil { - t.Fatalf("read detached child PID: %v", err) - } - pid, err := strconv.Atoi(string(pidData)) - if err != nil { - t.Fatalf("parse detached child PID %q: %v", pidData, err) - } - t.Cleanup(func() { - if !processIsActive(pid) { - return - } - process, findErr := os.FindProcess(pid) - if findErr != nil { - t.Errorf("find detached child %d: %v", pid, findErr) - return - } - if killErr := process.Kill(); killErr != nil { - t.Errorf("kill detached child %d: %v", pid, killErr) - return - } - awaitProcessExit(t, pid) - }) - if !processIsActive(pid) { + if !child.running() { t.Fatalf("successful RunCommand terminated detached child %d", pid) } } diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go index eb00dfef1..02689bed6 100644 --- a/internal/execution/command_tree_unix.go +++ b/internal/execution/command_tree_unix.go @@ -4,38 +4,89 @@ package execution import ( "errors" + "io" "os" "os/exec" + "sync" "syscall" ) type commandTree struct { - ready chan struct{} - pid int + mu sync.Mutex + ready chan struct{} + readyOnce sync.Once + groupID int + anchor *exec.Cmd + anchorInput io.WriteCloser + signal func(int, syscall.Signal) error + canceled bool + cancelErr error + closed bool } func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - ConfigureProcessGroup(command) - return &commandTree{ready: make(chan struct{})}, nil -} + // Keep the group leader alive until cleanup so an exited command cannot + // leave a reusable PID as the only identity for its live descendants. + anchor := exec.Command("/bin/sh", "-c", "read _") + anchor.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + anchorInput, err := anchor.StdinPipe() + if err != nil { + return nil, err + } + if err := anchor.Start(); err != nil { + _ = anchorInput.Close() + return nil, err + } -func (tree *commandTree) attach(process *os.Process) error { - if process != nil { - tree.pid = process.Pid + tree := &commandTree{ + ready: make(chan struct{}), + groupID: anchor.Process.Pid, + anchor: anchor, + anchorInput: anchorInput, + signal: syscall.Kill, } - close(tree.ready) + if command.SysProcAttr == nil { + command.SysProcAttr = &syscall.SysProcAttr{} + } + command.SysProcAttr.Setpgid = true + command.SysProcAttr.Pgid = tree.groupID + return tree, nil +} + +func (tree *commandTree) attach(*os.Process) error { + tree.readyOnce.Do(func() { close(tree.ready) }) return nil } func (tree *commandTree) cancel() error { <-tree.ready - if tree.pid <= 1 { + tree.mu.Lock() + defer tree.mu.Unlock() + if tree.closed || tree.canceled || tree.groupID <= 1 { + return tree.cancelErr + } + tree.canceled = true + if err := tree.signal(-tree.groupID, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + tree.cancelErr = err + } + return tree.cancelErr +} + +func (tree *commandTree) close() error { + tree.mu.Lock() + defer tree.mu.Unlock() + if tree.closed { return nil } - if err := syscall.Kill(-tree.pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { - return err + tree.closed = true + if tree.anchorInput != nil { + _ = tree.anchorInput.Close() + tree.anchorInput = nil + } + if tree.anchor != nil { + _ = tree.anchor.Wait() + tree.anchor = nil } + tree.groupID = 0 return nil } - -func (*commandTree) close() error { return nil } diff --git a/internal/execution/command_tree_unix_test.go b/internal/execution/command_tree_unix_test.go new file mode 100644 index 000000000..289ed70c5 --- /dev/null +++ b/internal/execution/command_tree_unix_test.go @@ -0,0 +1,174 @@ +//go:build !windows + +package execution + +import ( + "context" + "errors" + "os/exec" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +func TestRunCommandAbsolutePathWithEmptyPATH(t *testing.T) { + t.Setenv("PATH", "") + ctx := context.Background() + command := exec.CommandContext(ctx, "/bin/sh", "-c", "exit 0") + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand with absolute executable and empty PATH: %v", err) + } +} + +func TestPrepareCommandTreeRetainsGroupIdentity(t *testing.T) { + attributes := &syscall.SysProcAttr{Setsid: true} + command := exec.Command("sh", "-c", "exit 7") + command.SysProcAttr = attributes + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepare command tree: %v", err) + } + defer tree.close() + + if command.SysProcAttr != attributes { + t.Fatal("prepareCommandTree replaced existing SysProcAttr") + } + if !attributes.Setsid || !attributes.Setpgid || attributes.Pgid != tree.groupID { + t.Fatalf("command attributes = %#v, want preserved Setsid and group %d", attributes, tree.groupID) + } + if pgid, err := syscall.Getpgid(tree.anchor.Process.Pid); err != nil || pgid != tree.groupID { + t.Fatalf("anchor process group = %d, %v; want %d", pgid, err, tree.groupID) + } + + // Setsid and joining an existing process group are intentionally incompatible; + // it is retained above only to verify that unrelated caller fields survive. + attributes.Setsid = false + if err := command.Start(); err != nil { + t.Fatalf("start command: %v", err) + } + if err := tree.attach(command.Process); err != nil { + t.Fatalf("attach command: %v", err) + } + if pgid, err := syscall.Getpgid(command.Process.Pid); err != nil || pgid != tree.groupID { + t.Fatalf("command process group = %d, %v; want %d", pgid, err, tree.groupID) + } + if err := command.Wait(); err == nil { + t.Fatal("command unexpectedly succeeded") + } + if err := syscall.Kill(tree.anchor.Process.Pid, 0); err != nil { + t.Fatalf("anchor did not retain group identity after command exit: %v", err) + } +} + +func TestCommandTreeCancelSignalsOnce(t *testing.T) { + ready := make(chan struct{}) + close(ready) + wantErr := errors.New("signal failed") + var calls atomic.Int32 + tree := &commandTree{ + ready: ready, + groupID: 123, + signal: func(pid int, signal syscall.Signal) error { + calls.Add(1) + if pid != -123 || signal != syscall.SIGKILL { + t.Errorf("signal target = (%d, %v), want (-123, SIGKILL)", pid, signal) + } + return wantErr + }, + } + + const callers = 32 + var wait sync.WaitGroup + wait.Add(callers) + for range callers { + go func() { + defer wait.Done() + if err := tree.cancel(); !errors.Is(err, wantErr) { + t.Errorf("cancel error = %v, want %v", err, wantErr) + } + }() + } + wait.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("signal calls = %d, want 1", got) + } +} + +func TestCommandTreeCloseWaitsForCancelAndPreventsLaterSignals(t *testing.T) { + command := exec.Command("sh", "-c", "exit 0") + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepare command tree: %v", err) + } + if err := tree.attach(nil); err != nil { + t.Fatalf("attach command tree: %v", err) + } + anchorPID := tree.anchor.Process.Pid + + signalStarted := make(chan struct{}) + releaseSignal := make(chan struct{}) + var calls atomic.Int32 + tree.signal = func(int, syscall.Signal) error { + calls.Add(1) + close(signalStarted) + <-releaseSignal + return nil + } + cancelDone := make(chan error, 1) + go func() { cancelDone <- tree.cancel() }() + <-signalStarted + + closeDone := make(chan error, 1) + go func() { closeDone <- tree.close() }() + select { + case err := <-closeDone: + t.Fatalf("close returned while signal was in flight: %v", err) + case <-time.After(100 * time.Millisecond): + } + if err := syscall.Kill(anchorPID, 0); err != nil { + t.Fatalf("anchor was released while signal was in flight: %v", err) + } + + close(releaseSignal) + if err := <-cancelDone; err != nil { + t.Fatalf("cancel command tree: %v", err) + } + if err := <-closeDone; err != nil { + t.Fatalf("close command tree: %v", err) + } + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after close: %v", err) + } + if err := tree.close(); err != nil { + t.Fatalf("repeated close: %v", err) + } + if got := calls.Load(); got != 1 { + t.Fatalf("signal calls after close = %d, want 1", got) + } +} + +func TestCommandTreeCancelAfterCloseDoesNotSignal(t *testing.T) { + ready := make(chan struct{}) + close(ready) + var calls atomic.Int32 + tree := &commandTree{ + ready: ready, + groupID: 123, + signal: func(int, syscall.Signal) error { + calls.Add(1) + return nil + }, + } + + if err := tree.close(); err != nil { + t.Fatalf("close command tree: %v", err) + } + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after close: %v", err) + } + if got := calls.Load(); got != 0 { + t.Fatalf("signal calls after close = %d, want 0", got) + } +} diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go index 06caa260c..5c6969f20 100644 --- a/internal/execution/command_tree_windows_test.go +++ b/internal/execution/command_tree_windows_test.go @@ -19,46 +19,58 @@ func TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails(t *testi switch os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") - child.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child", + "ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE="+os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), 30*time.Second) return } + root := t.TempDir() + stopFile := filepath.Join(root, "stop") originalAssign := assignCommandProcessToJob - assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + commandOwner := ownHelperHandle(t, stopFile) + assignCommandProcessToJob = func(_ windows.Handle, process windows.Handle) error { + if err := commandOwner.retain(process); err != nil { + return err + } return windows.ERROR_ACCESS_DENIED } t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - pidFile := filepath.Join(t.TempDir(), "child.pid") - ctx := context.Background() + pidFile := filepath.Join(root, "child.pid") + escapedChild := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") command.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=root", "ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE="+pidFile, + "ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE="+stopFile, ) - started := time.Now() - err := RunCommand(ctx, command) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand took %s after job assignment failed", elapsed) - } + err := waitForRunCommand(t, runCommandAsync(ctx, command), 4*time.Second) if !errors.Is(err, windows.ERROR_ACCESS_DENIED) { t.Fatalf("RunCommand error = %v, want ERROR_ACCESS_DENIED", err) } if command.ProcessState == nil || !command.ProcessState.Exited() { t.Fatalf("suspended command was not killed and reaped: state = %v", command.ProcessState) } - if _, statErr := os.Stat(pidFile); !errors.Is(statErr, os.ErrNotExist) { - t.Fatalf("suspended command spawned a descendant after job assignment failed: PID file error = %v", statErr) + observed, observeErr := escapedChild.observeReady() + _, statErr := os.Stat(pidFile) + if observeErr != nil || observed || !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("suspended command spawned a descendant after job assignment failed: observed = %t, observation error = %v, PID file error = %v", observed, observeErr, statErr) } } diff --git a/internal/execution/exit_error.go b/internal/execution/exit_error.go new file mode 100644 index 000000000..eab627889 --- /dev/null +++ b/internal/execution/exit_error.go @@ -0,0 +1,31 @@ +package execution + +import "os/exec" + +// AsPureExitError reports whether err is an ordinary process exit or a join +// tree containing only ordinary process exits. It does not unwrap single-error +// wrappers, which may carry a distinct lifecycle or cleanup failure. +func AsPureExitError(err error) (*exec.ExitError, bool) { + if exitErr, ok := err.(*exec.ExitError); ok { + return exitErr, exitErr != nil + } + joined, ok := err.(interface{ Unwrap() []error }) + if !ok { + return nil, false + } + causes := joined.Unwrap() + if len(causes) == 0 { + return nil, false + } + var first *exec.ExitError + for _, cause := range causes { + exitErr, ok := AsPureExitError(cause) + if !ok { + return nil, false + } + if first == nil { + first = exitErr + } + } + return first, first != nil +} diff --git a/internal/execution/exit_error_test.go b/internal/execution/exit_error_test.go new file mode 100644 index 000000000..323a87ad7 --- /dev/null +++ b/internal/execution/exit_error_test.go @@ -0,0 +1,41 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "os/exec" + "testing" +) + +func TestAsPureExitError(t *testing.T) { + first := &exec.ExitError{} + second := &exec.ExitError{} + var nilExit *exec.ExitError + tests := []struct { + name string + err error + want *exec.ExitError + ok bool + }{ + {name: "nil"}, + {name: "direct", err: first, want: first, ok: true}, + {name: "joined", err: errors.Join(first, second), want: first, ok: true}, + {name: "nested joins", err: errors.Join(errors.Join(first, second), &exec.ExitError{}), want: first, ok: true}, + {name: "join with nil", err: errors.Join(first, nil), want: first, ok: true}, + {name: "ordinary error", err: errors.New("start failed")}, + {name: "mixed join", err: errors.Join(first, context.Canceled)}, + {name: "nested mixed join", err: errors.Join(first, errors.Join(second, context.DeadlineExceeded))}, + {name: "wrapped exit", err: fmt.Errorf("cleanup failed: %w", first)}, + {name: "join containing wrapped exit", err: errors.Join(first, fmt.Errorf("wrapped: %w", second))}, + {name: "typed nil exit", err: nilExit}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := AsPureExitError(test.err) + if got != test.want || ok != test.ok { + t.Fatalf("AsPureExitError(%v) = (%p, %v), want (%p, %v)", test.err, got, ok, test.want, test.ok) + } + }) + } +} diff --git a/internal/execution/runner.go b/internal/execution/runner.go index 9e3ecbf9a..7da175419 100644 --- a/internal/execution/runner.go +++ b/internal/execution/runner.go @@ -174,8 +174,7 @@ func commandExitCode(err error) int { if err == nil { return 0 } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := AsPureExitError(err); ok { return exitErr.ExitCode() } return -1 diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index 3b5c998ab..f7bf6e6de 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "os" "os/exec" "strings" @@ -311,8 +310,7 @@ func execCommandRunner(ctx context.Context, command string, args []string, stdin if err == nil { return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 4ce1b9ba7..7e4de503b 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -30,26 +30,45 @@ func beforeToolConfig(hooks ...Definition) Config { func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { switch os.Getenv("ZERO_HOOK_TREE_HELPER") { case "parent": + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PARENT_PID_FILE"), []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { + os.Exit(2) + } child := exec.Command(os.Args[0], "-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$") - child.Env = append(os.Environ(), "ZERO_HOOK_TREE_HELPER=grandchild") + child.Env = append(os.Environ(), + "ZERO_HOOK_TREE_HELPER=grandchild", + "ZERO_HOOK_TREE_STOP_FILE="+os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { - os.Exit(2) - } - if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { os.Exit(3) } - select {} + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_GRANDCHILD_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_READY_FILE"), []byte("ready"), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(5) + } + waitForHookTreeStop(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), 30*time.Second) + _ = child.Wait() + return case "grandchild": - time.Sleep(30 * time.Second) + waitForHookTreeStop(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), 30*time.Second) os.Exit(0) } - pidFile := filepath.Join(t.TempDir(), "grandchild.pid") - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + root := t.TempDir() + parentPIDFile := filepath.Join(root, "parent.pid") + grandchildPIDFile := filepath.Join(root, "grandchild.pid") + readyFile := filepath.Join(root, "ready") + stopFile := filepath.Join(root, "stop") + owner := newHookTestProcessOwner(t, stopFile, parentPIDFile, grandchildPIDFile) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - started := time.Now() resultChannel := make(chan commandResult, 1) go func() { resultChannel <- execCommandRunner( @@ -58,15 +77,23 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, nil, "", - append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent", "ZERO_HOOK_TREE_PID_FILE="+pidFile), + append(os.Environ(), + "ZERO_HOOK_TREE_HELPER=parent", + "ZERO_HOOK_TREE_PARENT_PID_FILE="+parentPIDFile, + "ZERO_HOOK_TREE_GRANDCHILD_PID_FILE="+grandchildPIDFile, + "ZERO_HOOK_TREE_READY_FILE="+readyFile, + "ZERO_HOOK_TREE_STOP_FILE="+stopFile, + ), ) }() + parentPID, grandchildPID := awaitHookTreeReady(t, owner, readyFile, parentPIDFile, grandchildPIDFile) + started := time.Now() var result commandResult select { case result = <-resultChannel: - case <-time.After(4 * time.Second): + case <-time.After(6 * time.Second): cancel() - t.Fatal("execCommandRunner did not return within four seconds after its timeout") + t.Fatal("execCommandRunner did not return within six seconds after its timeout") } if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) @@ -74,15 +101,50 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { if result.Err == nil && result.ExitCode == 0 { t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) } - pidData, err := os.ReadFile(pidFile) - if err != nil { - t.Fatalf("read grandchild PID: %v", err) + for role, pid := range map[string]int{"parent": parentPID, "grandchild": grandchildPID} { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Fatalf("%s process %d survived hook cancellation: %v", role, pid, err) + } } - pid, err := strconv.Atoi(string(pidData)) - if err != nil { - t.Fatalf("parse grandchild PID %q: %v", pidData, err) +} + +func waitForHookTreeStop(stopFile string, lifetime time.Duration) { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } +} + +func awaitHookTreeReady(t *testing.T, owner *hookTestProcessOwner, readyFile string, pidFiles ...string) (int, int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(readyFile); err == nil { + pids := make([]int, 0, len(pidFiles)) + for _, path := range pidFiles { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read helper PID handoff %q: %v", path, err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatalf("parse helper PID handoff %q: %v", data, err) + } + if err := owner.retain(pid); err != nil { + t.Fatalf("retain helper process %d: %v", pid, err) + } + pids = append(pids, pid) + } + return pids[0], pids[1] + } + if time.Now().After(deadline) { + t.Fatal("process-tree helper did not hand off parent and grandchild identities") + } + time.Sleep(10 * time.Millisecond) } - awaitHookProcessExit(t, pid) } func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { diff --git a/internal/hooks/process_test_unix.go b/internal/hooks/process_test_unix.go deleted file mode 100644 index 01e22f2a8..000000000 --- a/internal/hooks/process_test_unix.go +++ /dev/null @@ -1,28 +0,0 @@ -//go:build !windows - -package hooks - -import ( - "errors" - "syscall" - "testing" - "time" -) - -func awaitHookProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for { - err := syscall.Kill(pid, syscall.Signal(0)) - if errors.Is(err, syscall.ESRCH) { - return - } - if err != nil { - t.Fatalf("probe grandchild process %d: %v", pid, err) - } - if time.Now().After(deadline) { - t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} diff --git a/internal/hooks/process_test_windows.go b/internal/hooks/process_test_windows.go deleted file mode 100644 index 2b859f545..000000000 --- a/internal/hooks/process_test_windows.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build windows - -package hooks - -import ( - "testing" - "time" - - "golang.org/x/sys/windows" -) - -const processStillActive = 259 - -func awaitHookProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for hookProcessIsActive(pid) { - if time.Now().After(deadline) { - t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} - -func hookProcessIsActive(pid int) bool { - handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) - if err != nil { - return false - } - defer windows.CloseHandle(handle) - var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { - return false - } - return exitCode == processStillActive -} diff --git a/internal/hooks/process_unix_test.go b/internal/hooks/process_unix_test.go new file mode 100644 index 000000000..af6e6642f --- /dev/null +++ b/internal/hooks/process_unix_test.go @@ -0,0 +1,81 @@ +//go:build !windows + +package hooks + +import ( + "errors" + "os" + "strconv" + "syscall" + "testing" + "time" +) + +type hookTestProcessOwner struct { + pids map[int]struct{} + exited map[int]struct{} + stopFile string + pidFiles []string +} + +func newHookTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *hookTestProcessOwner { + t.Helper() + owner := &hookTestProcessOwner{pids: make(map[int]struct{}), exited: make(map[int]struct{}), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *hookTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + owner.pids[pid] = struct{}{} + return nil +} + +func (owner *hookTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + delete(owner.pids, pid) + owner.exited[pid] = struct{}{} + return nil + } + if err != nil { + return err + } + if time.Now().After(deadline) { + return errors.New("process is still running") + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *hookTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid := range owner.pids { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + } +} + +func (owner *hookTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil && pid > 0 && pid != os.Getpid() { + if _, exited := owner.exited[pid]; !exited { + owner.pids[pid] = struct{}{} + } + } + } +} diff --git a/internal/hooks/process_windows_test.go b/internal/hooks/process_windows_test.go new file mode 100644 index 000000000..06796c923 --- /dev/null +++ b/internal/hooks/process_windows_test.go @@ -0,0 +1,95 @@ +//go:build windows + +package hooks + +import ( + "errors" + "fmt" + "os" + "strconv" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type hookTestProcessOwner struct { + handles map[int]windows.Handle + stopFile string + pidFiles []string +} + +func newHookTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *hookTestProcessOwner { + t.Helper() + owner := &hookTestProcessOwner{handles: make(map[int]windows.Handle), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *hookTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + if _, ok := owner.handles[pid]; ok { + return nil + } + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return err + } + owner.handles[pid] = handle + return nil +} + +func (owner *hookTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + handle, ok := owner.handles[pid] + if !ok { + return errors.New("test process identity was not retained") + } + wait, err := windows.WaitForSingleObject(handle, uint32(timeout/time.Millisecond)) + if err != nil { + return err + } + if wait != windows.WAIT_OBJECT_0 { + return fmt.Errorf("process is still running (wait result %#x)", wait) + } + return nil +} + +func (owner *hookTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid, handle := range owner.handles { + wait, err := windows.WaitForSingleObject(handle, 2_000) + if err == nil && wait == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(handle, 1); err != nil { + t.Errorf("terminate test process %d: %v", pid, err) + } + } + } + for pid, handle := range owner.handles { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + if err := windows.CloseHandle(handle); err != nil { + t.Errorf("close test process %d handle: %v", pid, err) + } + delete(owner.handles, pid) + } +} + +func (owner *hookTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil { + _ = owner.retain(pid) + } + } +} diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index a922bd998..73c3335bb 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -338,8 +338,11 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { cmd.Stdout = &stdout cmd.Stderr = &stderr runErr := execution.RunCommand(ctx, cmd) - if errors.Is(runErr, exec.ErrWaitDelay) { - return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + if !runEndCanReconcile(runErr) { + if errors.Is(runErr, exec.ErrWaitDelay) { + return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + } + return TaskOutcome{Err: fmt.Errorf("zero exec command failed: %w", runErr)} } // The terminal run_end exit code is authoritative for pass/fail: a non-zero @@ -370,6 +373,17 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { } } +// runEndCanReconcile reports whether a command result contains only an ordinary +// process exit status. A run_end may explain success or an *exec.ExitError, but +// it must not hide cancellation, startup, process-tree, or output-cleanup errors. +func runEndCanReconcile(err error) bool { + if err == nil { + return true + } + _, ok := execution.AsPureExitError(err) + return ok +} + func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { args := []string{"exec", "--output-format", "stream-json"} if model := strings.TrimSpace(rc.Model); model != "" { diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index e42d0af82..f01667798 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -3,12 +3,15 @@ package perfbench import ( "context" "errors" + "fmt" "os" + "os/exec" "path/filepath" "runtime" "strconv" "strings" "testing" + "time" ) func sampleTaskSet() TaskSet { @@ -276,6 +279,40 @@ func writeExecStub(t *testing.T, body string) string { return path } +func writeBlockingExecStub(t *testing.T) string { + t.Helper() + dir := t.TempDir() + source := filepath.Join(dir, "main.go") + if err := os.WriteFile(source, []byte(`package main + +import ( + "fmt" + "os" + "time" +) + +func main() { + fmt.Println("{\"type\":\"run_end\",\"exitCode\":0}") + if ready := os.Getenv("PERFBENCH_BLOCKING_STUB_READY"); ready != "" { + if err := os.WriteFile(ready, nil, 0600); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) +} +`), 0o600); err != nil { + t.Fatalf("write blocking exec stub: %v", err) + } + binary := filepath.Join(dir, "zero-stub") + if runtime.GOOS == "windows" { + binary += ".exe" + } + if output, err := exec.Command("go", "build", "-o", binary, source).CombinedOutput(); err != nil { + t.Fatalf("build blocking exec stub: %v\n%s", err, output) + } + return binary +} + func TestNewExecRunnerNonZeroRunEndIsFailNotError(t *testing.T) { // A non-zero run_end exit code is a normal task failure, not a harness error, // even though the process itself exits non-zero. @@ -292,6 +329,78 @@ exit 1 } } +func TestRunEndCanReconcile(t *testing.T) { + exitErr := &exec.ExitError{} + tests := []struct { + name string + err error + want bool + }{ + {name: "success", want: true}, + {name: "exit error", err: exitErr, want: true}, + {name: "joined exit errors", err: errors.Join(exitErr, &exec.ExitError{}), want: true}, + {name: "ordinary error", err: errors.New("startup failed")}, + {name: "canceled", err: context.Canceled}, + {name: "deadline", err: context.DeadlineExceeded}, + {name: "wait delay", err: exec.ErrWaitDelay}, + {name: "exit plus cancellation", err: errors.Join(exitErr, context.Canceled)}, + {name: "wrapped exit error", err: fmt.Errorf("attachment failed: %w", exitErr)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := runEndCanReconcile(test.err); got != test.want { + t.Fatalf("runEndCanReconcile(%v) = %v, want %v", test.err, got, test.want) + } + }) + } +} + +func TestNewExecRunnerRunEndCannotHideContextFailure(t *testing.T) { + stub := writeBlockingExecStub(t) + tests := []struct { + name string + context func(t *testing.T) context.Context + wantErr error + }{ + { + name: "cancellation", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + timer := time.AfterFunc(time.Second, cancel) + t.Cleanup(func() { timer.Stop() }) + return ctx + }, + wantErr: context.Canceled, + }, + { + name: "deadline", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + return ctx + }, + wantErr: context.DeadlineExceeded, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ready := filepath.Join(t.TempDir(), "ready") + t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) + outcome := NewExecRunner(stub)(test.context(t), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + if _, err := os.Stat(ready); err != nil { + t.Fatalf("stub did not emit run_end before the context failed: %v", err) + } + if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { + t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) + } + if outcome.Passed { + t.Fatal("context failure must not reach task pass accounting") + } + }) + } +} + func TestNewExecRunnerMissingRunEndFailsClosed(t *testing.T) { // A clean exit with no terminal run_end event is a harness error: we cannot // claim the task passed when the agent never reported a terminal event. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 1e30db2cf..ab4b5f87d 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -669,12 +669,18 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { runErr := execution.RunCommand(ctx, cmd) wallMs := float64(time.Since(start).Microseconds()) / 1000 - exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) outcome := TurnTaskOutcome{WallMs: wallMs} - if errors.Is(runErr, exec.ErrWaitDelay) { - outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + if !runEndCanReconcile(runErr) { + if errors.Is(runErr, exec.ErrWaitDelay) { + outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + } else { + outcome.Err = fmt.Errorf("zero exec command failed: %w", runErr) + } return outcome - } else if haveExit && exitCode != 0 { + } + + exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) + if haveExit && exitCode != 0 { outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode) } else if !haveExit { detail := strings.TrimSpace(errBuf.String()) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 2be6d1a1f..ab9f372d4 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -657,6 +657,53 @@ exit 0 } } +func TestNewTurnExecRunnerRunEndCannotHideContextFailure(t *testing.T) { + task := BenchTask{ID: "context-failure", Prompt: "p", WorkspaceFixture: t.TempDir()} + stub := writeBlockingExecStub(t) + tests := []struct { + name string + context func(t *testing.T) context.Context + wantErr error + }{ + { + name: "cancellation", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + timer := time.AfterFunc(time.Second, cancel) + t.Cleanup(func() { timer.Stop() }) + return ctx + }, + wantErr: context.Canceled, + }, + { + name: "deadline", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + return ctx + }, + wantErr: context.DeadlineExceeded, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ready := filepath.Join(t.TempDir(), "ready") + t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) + outcome := NewTurnExecRunner(stub)(test.context(t), task, RunContext{Model: "m"}) + if _, err := os.Stat(ready); err != nil { + t.Fatalf("stub did not emit run_end before the context failed: %v", err) + } + if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { + t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) + } + if outcome.Passed || outcome.VerifyErr != "" { + t.Fatalf("context failure must precede oracle accounting, got %#v", outcome) + } + }) + } +} + // assertVerifyFailed asserts an outcome failed specifically because the oracle // rejected the work — Passed is false, there is no harness error (Err nil), and // VerifyErr carries the surfaced failure detail. This is stronger than merely @@ -810,6 +857,7 @@ func TestOracleAuthoritativeOnIncompleteExit(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, `sed 's/const MaxRetries = 3/const RetryLimit = 3/' main.go > .zero-tmp && mv .zero-tmp main.go echo '{"type":"run_end","exitCode":4}' +exit 4 `) if outcome.Err != nil { t.Fatalf("incomplete-exit with a correct edit should pass, got harness error: %v", outcome.Err) @@ -836,7 +884,8 @@ func TestNonIncompleteExitStaysAuthoritative(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, fmt.Sprintf(`sed 's/const MaxRetries = 3/const RetryLimit = 3/' main.go > .zero-tmp && mv .zero-tmp main.go echo '{"type":"run_end","exitCode":%d}' -`, code)) +exit %d +`, code, code)) if outcome.Err != nil { t.Fatalf("a nonzero exit should be a task fail, not a harness error: %v", outcome.Err) } @@ -860,6 +909,7 @@ echo '{"type":"run_end","exitCode":%d}' func TestIncompleteExitStillFailsWhenOracleFails(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, `echo '{"type":"run_end","exitCode":4}' +exit 4 `) assertVerifyFailed(t, "incomplete exit with no edit applied", outcome) } @@ -872,6 +922,7 @@ func TestIncompleteExitStillFailsWhenOracleFails(t *testing.T) { func TestNonzeroExitStillFailsLatencyOnly(t *testing.T) { task := loadBaselineTask(t, "longproc-01") outcome := runTurnStub(t, task, `echo '{"type":"run_end","exitCode":4}' +exit 4 `) if outcome.Err != nil { t.Fatalf("latency-only nonzero exit should be a verify fail, not a harness error: %v", outcome.Err) diff --git a/internal/verify/process_unix_test.go b/internal/verify/process_unix_test.go new file mode 100644 index 000000000..cdd5ff7f4 --- /dev/null +++ b/internal/verify/process_unix_test.go @@ -0,0 +1,81 @@ +//go:build !windows + +package verify + +import ( + "errors" + "os" + "strconv" + "syscall" + "testing" + "time" +) + +type verifyTestProcessOwner struct { + pids map[int]struct{} + exited map[int]struct{} + stopFile string + pidFiles []string +} + +func newVerifyTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *verifyTestProcessOwner { + t.Helper() + owner := &verifyTestProcessOwner{pids: make(map[int]struct{}), exited: make(map[int]struct{}), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *verifyTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + owner.pids[pid] = struct{}{} + return nil +} + +func (owner *verifyTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + delete(owner.pids, pid) + owner.exited[pid] = struct{}{} + return nil + } + if err != nil { + return err + } + if time.Now().After(deadline) { + return errors.New("process is still running") + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *verifyTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid := range owner.pids { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + } +} + +func (owner *verifyTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil && pid > 0 && pid != os.Getpid() { + if _, exited := owner.exited[pid]; !exited { + owner.pids[pid] = struct{}{} + } + } + } +} diff --git a/internal/verify/process_windows_test.go b/internal/verify/process_windows_test.go new file mode 100644 index 000000000..a8612a25e --- /dev/null +++ b/internal/verify/process_windows_test.go @@ -0,0 +1,95 @@ +//go:build windows + +package verify + +import ( + "errors" + "fmt" + "os" + "strconv" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type verifyTestProcessOwner struct { + handles map[int]windows.Handle + stopFile string + pidFiles []string +} + +func newVerifyTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *verifyTestProcessOwner { + t.Helper() + owner := &verifyTestProcessOwner{handles: make(map[int]windows.Handle), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *verifyTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + if _, ok := owner.handles[pid]; ok { + return nil + } + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return err + } + owner.handles[pid] = handle + return nil +} + +func (owner *verifyTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + handle, ok := owner.handles[pid] + if !ok { + return errors.New("test process identity was not retained") + } + wait, err := windows.WaitForSingleObject(handle, uint32(timeout/time.Millisecond)) + if err != nil { + return err + } + if wait != windows.WAIT_OBJECT_0 { + return fmt.Errorf("process is still running (wait result %#x)", wait) + } + return nil +} + +func (owner *verifyTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid, handle := range owner.handles { + wait, err := windows.WaitForSingleObject(handle, 2_000) + if err == nil && wait == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(handle, 1); err != nil { + t.Errorf("terminate test process %d: %v", pid, err) + } + } + } + for pid, handle := range owner.handles { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + if err := windows.CloseHandle(handle); err != nil { + t.Errorf("close test process %d handle: %v", pid, err) + } + delete(owner.handles, pid) + } +} + +func (owner *verifyTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil { + _ = owner.retain(pid) + } + } +} diff --git a/internal/verify/verify.go b/internal/verify/verify.go index a571cada8..3464d0f55 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -3,7 +3,6 @@ package verify import ( "bytes" "context" - "errors" "fmt" "os" "os/exec" @@ -310,8 +309,7 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti exitCode := 0 if err != nil { exitCode = -1 - var exitError *exec.ExitError - if errors.As(err, &exitError) { + if exitError, ok := execution.AsPureExitError(err); ok { exitCode = exitError.ExitCode() err = nil } diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go index b8792ca09..2e49e1634 100644 --- a/internal/verify/verify_test.go +++ b/internal/verify/verify_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "time" @@ -16,36 +17,115 @@ import ( func TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { switch os.Getenv("ZERO_VERIFY_TREE_HELPER") { case "parent": - if err := os.Setenv("ZERO_VERIFY_TREE_HELPER", "grandchild"); err != nil { + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_PARENT_PID_FILE"), []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { os.Exit(2) } child := exec.Command(os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$") - child.Env = os.Environ() + child.Env = append(os.Environ(), + "ZERO_VERIFY_TREE_HELPER=grandchild", + "ZERO_VERIFY_TREE_STOP_FILE="+os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(3) } - select {} + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_GRANDCHILD_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_READY_FILE"), []byte("ready"), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(5) + } + waitForVerifyTreeStop(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), 30*time.Second) + _ = child.Wait() + return case "grandchild": - time.Sleep(30 * time.Second) + waitForVerifyTreeStop(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), 30*time.Second) return } + root := t.TempDir() + parentPIDFile := filepath.Join(root, "parent.pid") + grandchildPIDFile := filepath.Join(root, "grandchild.pid") + readyFile := filepath.Join(root, "ready") + stopFile := filepath.Join(root, "stop") + owner := newVerifyTestProcessOwner(t, stopFile, parentPIDFile, grandchildPIDFile) t.Setenv("ZERO_VERIFY_TREE_HELPER", "parent") - plan := Plan{Root: t.TempDir(), Checks: []Check{{ + t.Setenv("ZERO_VERIFY_TREE_PARENT_PID_FILE", parentPIDFile) + t.Setenv("ZERO_VERIFY_TREE_GRANDCHILD_PID_FILE", grandchildPIDFile) + t.Setenv("ZERO_VERIFY_TREE_READY_FILE", readyFile) + t.Setenv("ZERO_VERIFY_TREE_STOP_FILE", stopFile) + plan := Plan{Root: root, Checks: []Check{{ ID: "tree.timeout", Name: "process tree timeout", Command: []string{os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$"}, }}} + reportChannel := make(chan Report, 1) + go func() { + reportChannel <- Run(context.Background(), plan, RunOptions{TimeoutMS: 3000}) + }() + parentPID, grandchildPID := awaitVerifyTreeReady(t, owner, readyFile, parentPIDFile, grandchildPIDFile) started := time.Now() - report := Run(context.Background(), plan, RunOptions{TimeoutMS: 100}) - if elapsed := time.Since(started); elapsed > 2*time.Second { + var report Report + select { + case report = <-reportChannel: + case <-time.After(6 * time.Second): + t.Fatal("defaultRunner did not return within six seconds after its timeout") + } + if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("defaultRunner remained blocked by grandchild output handles for %s", elapsed) } if report.OK || len(report.Results) != 1 || report.Results[0].Status == StatusPass { t.Fatalf("timed-out defaultRunner command unexpectedly passed: %#v", report) } + for role, pid := range map[string]int{"parent": parentPID, "grandchild": grandchildPID} { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Fatalf("%s process %d survived verify cancellation: %v", role, pid, err) + } + } +} + +func waitForVerifyTreeStop(stopFile string, lifetime time.Duration) { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } +} + +func awaitVerifyTreeReady(t *testing.T, owner *verifyTestProcessOwner, readyFile string, pidFiles ...string) (int, int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(readyFile); err == nil { + pids := make([]int, 0, len(pidFiles)) + for _, path := range pidFiles { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read helper PID handoff %q: %v", path, err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatalf("parse helper PID handoff %q: %v", data, err) + } + if err := owner.retain(pid); err != nil { + t.Fatalf("retain helper process %d: %v", pid, err) + } + pids = append(pids, pid) + } + return pids[0], pids[1] + } + if time.Now().After(deadline) { + t.Fatal("process-tree helper did not hand off parent and grandchild identities") + } + time.Sleep(10 * time.Millisecond) + } } func TestDetectPlanFindsBunAndGoChecks(t *testing.T) { From aac2bd9116ce346d6d7784b83fd7481f45e5988e Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Thu, 27 Aug 2026 22:18:09 +0200 Subject: [PATCH 08/17] fix(process): bound context command cleanup Amp-Thread-ID: https://ampcode.com/threads/T-01a0448f-5860-721c-8a47-5119fc57f685 Co-authored-by: Amp Co-authored-by: Pierre Bruno --- internal/agenteval/agent_command.go | 3 ++ internal/agenteval/materialize.go | 3 ++ internal/agenteval/run.go | 4 ++ internal/dictation/runner.go | 3 ++ internal/execution/command_context.go | 22 +++++++++ internal/hooks/dispatch.go | 1 + internal/hooks/dispatch_test.go | 36 +++++++++++++++ internal/perfbench/perfbench.go | 64 +++++++-------------------- internal/perfbench/taskbench.go | 4 ++ internal/perfbench/turn_bench.go | 2 + internal/verify/verify.go | 2 + 11 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 internal/execution/command_context.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index a07d32ef2..dc4cf5197 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -6,6 +6,8 @@ import ( "errors" "os/exec" "strings" + + "github.com/Gitlawb/zero/internal/execution" ) type AgentRunInput struct { @@ -61,6 +63,7 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A } command := expandAgentCommand(runner.Command, input) cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = input.WorkspacePath stdout := &capWriter{limit: limit} stderr := &capWriter{limit: limit} diff --git a/internal/agenteval/materialize.go b/internal/agenteval/materialize.go index 71e683dfc..e5076eb3e 100644 --- a/internal/agenteval/materialize.go +++ b/internal/agenteval/materialize.go @@ -10,6 +10,8 @@ import ( "os/exec" "path/filepath" "strings" + + "github.com/Gitlawb/zero/internal/execution" ) type Materializer struct{} @@ -188,6 +190,7 @@ func initGitBaseline(ctx context.Context, workspace string) error { } for _, args := range commands { cmd := exec.CommandContext(ctx, "git", args...) + execution.HardenCommandContext(cmd) cmd.Dir = workspace cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=Zero Eval", diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index e6f13a500..2e8b53557 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -10,6 +10,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // defaultCommandTimeout bounds a single verification command so a hung command @@ -139,6 +141,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command } args := trimCommand(command.Command) cmd := exec.CommandContext(ctx, args[0], args[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = workspace var stdout bytes.Buffer var stderr bytes.Buffer @@ -167,6 +170,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command func defaultRunGit(ctx context.Context, workspace string, args ...string) ([]byte, error) { allArgs := append([]string{"-C", workspace}, args...) cmd := exec.CommandContext(ctx, "git", allArgs...) + execution.HardenCommandContext(cmd) output, err := cmd.Output() if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { diff --git a/internal/dictation/runner.go b/internal/dictation/runner.go index 3db6eb156..bdfe6362a 100644 --- a/internal/dictation/runner.go +++ b/internal/dictation/runner.go @@ -7,6 +7,8 @@ import ( "os" "os/exec" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // commandSpec describes one capture-process invocation. Argv is always @@ -104,6 +106,7 @@ func (p *realProcess) Kill() error { return p.cmd.Process.Kill() } func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) + execution.HardenCommandContext(cmd) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go new file mode 100644 index 000000000..7f21fa71a --- /dev/null +++ b/internal/execution/command_context.go @@ -0,0 +1,22 @@ +package execution + +import ( + "os/exec" +) + +// HardenCommandContext makes a context-bound command terminate its process +// tree and prevents inherited output handles from blocking Wait indefinitely. +// Call this before Start or Run. +func HardenCommandContext(command *exec.Cmd) { + if command == nil { + return + } + ConfigureProcessGroup(command) + command.WaitDelay = processWaitDelay + command.Cancel = func() error { + if command.Process == nil { + return nil + } + return KillProcessTree(command.Process.Pid) + } +} diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index d5bb13e3c..946ed4b3a 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -298,6 +298,7 @@ func (dispatcher *Dispatcher) recordCompleted(hook Definition, input DispatchInp // ExitCode (not Err); Err is reserved for commands that could not be launched. func execCommandRunner(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { cmd := exec.CommandContext(ctx, command, args...) + execution.HardenCommandContext(cmd) cmd.Dir = cwd cmd.Env = env if len(stdin) > 0 { diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 40d6e295f..099f3b5be 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -2,6 +2,7 @@ package hooks import ( "context" + "os" "os/exec" "path/filepath" "runtime" @@ -25,6 +26,41 @@ func beforeToolConfig(hooks ...Definition) Config { return Config{Enabled: true, Hooks: hooks} } +func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { + switch os.Getenv("ZERO_HOOK_TREE_HELPER") { + case "parent": + child := exec.Command(os.Args[0], "-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$") + child.Env = append(os.Environ(), "ZERO_HOOK_TREE_HELPER=grandchild") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + select {} + case "grandchild": + time.Sleep(30 * time.Second) + os.Exit(0) + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + result := execCommandRunner( + ctx, + os.Args[0], + []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, + nil, + "", + append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent"), + ) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) + } + if result.Err == nil && result.ExitCode == 0 { + t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) + } +} + func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { var calls []string runner := func(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { diff --git a/internal/perfbench/perfbench.go b/internal/perfbench/perfbench.go index 31afe7b69..762ee1ee9 100644 --- a/internal/perfbench/perfbench.go +++ b/internal/perfbench/perfbench.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/release" ) @@ -266,6 +267,7 @@ func MeasureColdStart(ctx context.Context, command []string) (float64, error) { } startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) output, err := cmd.CombinedOutput() durationMs := RoundMetric(float64(time.Since(startedAt).Microseconds()) / 1000) @@ -282,17 +284,9 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl rssBefore := readHarnessMemoryMb() startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Env = offlineBenchmarkEnv(os.Environ()) - stdout, err := cmd.StdoutPipe() - if err != nil { - return firstOutputSample{}, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return firstOutputSample{}, err - } - var once sync.Once var firstOutputAt time.Time markFirstOutput := func() { @@ -300,32 +294,19 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl firstOutputAt = time.Now() }) } - - if err := cmd.Start(); err != nil { - return firstOutputSample{}, err - } - stdoutChan := make(chan pipeResult, 1) - stderrChan := make(chan pipeResult, 1) - go readTimedPipe(stdout, markFirstOutput, stdoutChan) - go readTimedPipe(stderr, markFirstOutput, stderrChan) - - stdoutResult := <-stdoutChan - stderrResult := <-stderrChan - waitErr := cmd.Wait() + stdout := &timedBuffer{onFirstWrite: markFirstOutput} + stderr := &timedBuffer{onFirstWrite: markFirstOutput} + cmd.Stdout = stdout + cmd.Stderr = stderr + waitErr := cmd.Run() finishedAt := time.Now() - if stdoutResult.Err != nil { - return firstOutputSample{}, stdoutResult.Err - } - if stderrResult.Err != nil { - return firstOutputSample{}, stderrResult.Err - } if firstOutputAt.IsZero() { firstOutputAt = finishedAt } rssAfter := readHarnessMemoryMb() if waitErr != nil { - return firstOutputSample{}, commandError(command, waitErr, stdoutResult.Text, stderrResult.Text) + return firstOutputSample{}, commandError(command, waitErr, stdout.String(), stderr.String()) } return firstOutputSample{ FirstOutputMs: RoundMetric(float64(firstOutputAt.Sub(startedAt).Microseconds()) / 1000), @@ -421,29 +402,16 @@ func median(sortedSamples []float64) float64 { return RoundMetric((sortedSamples[middle-1] + sortedSamples[middle]) / 2) } -type pipeResult struct { - Text string - Err error +type timedBuffer struct { + bytes.Buffer + onFirstWrite func() } -func readTimedPipe(reader io.Reader, onFirstChunk func(), result chan<- pipeResult) { - var buffer bytes.Buffer - chunk := make([]byte, 32*1024) - for { - n, err := reader.Read(chunk) - if n > 0 { - onFirstChunk() - _, _ = buffer.Write(chunk[:n]) - } - if err != nil { - if errors.Is(err, io.EOF) { - result <- pipeResult{Text: buffer.String()} - return - } - result <- pipeResult{Text: buffer.String(), Err: err} - return - } +func (buffer *timedBuffer) Write(data []byte) (int, error) { + if len(data) > 0 { + buffer.onFirstWrite() } + return buffer.Buffer.Write(data) } func commandError(command []string, err error, stdout string, stderr string) error { diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index cfcab1a86..ea9a0d86f 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -13,6 +13,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/Gitlawb/zero/internal/execution" ) // TaskSchemaVersion is the schema version of a published task-benchmark result. @@ -328,6 +330,7 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { return func(ctx context.Context, task BenchTask, rc RunContext) TaskOutcome { args := buildExecArgs(task, rc, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -383,6 +386,7 @@ func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { func runVerification(ctx context.Context, task BenchTask) TaskOutcome { cmd := exec.CommandContext(ctx, task.VerificationCommand[0], task.VerificationCommand[1:]...) + execution.HardenCommandContext(cmd) // Inherit the environment so the verifier sees PATH, HOME, language toolchain // vars, etc. — the same surface a maintainer gets running the command by hand // (matching the agent run above). NO_COLOR is appended for stable output. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index a157b140b..788a95944 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -16,6 +16,7 @@ import ( "time" "github.com/Gitlawb/zero/internal/execprofile" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/trace" ) @@ -657,6 +658,7 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { args := buildTurnExecArgs(task, rc, tracePath, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) + execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir diff --git a/internal/verify/verify.go b/internal/verify/verify.go index 363cad6ba..cd152bf41 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/Gitlawb/zero/internal/execution" "github.com/Gitlawb/zero/internal/redaction" "github.com/Gitlawb/zero/internal/testrunner" ) @@ -299,6 +300,7 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti defer cancel() cmd := exec.CommandContext(commandCtx, command[0], command[1:]...) + execution.HardenCommandContext(cmd) cmd.Dir = dir var stdout bytes.Buffer var stderr bytes.Buffer From 54b457f74d377d6a90c13cc983cee4009a3c5365 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 27 Aug 2026 21:09:25 +0000 Subject: [PATCH 09/17] fix(process): retain command tree cancellation identity Amp-Thread-ID: https://ampcode.com/threads/T-01a044e2-92ad-774b-9a86-094e2e5293bf Co-authored-by: Pierre Bruno --- internal/agenteval/agent_command.go | 3 +- internal/agenteval/materialize.go | 3 +- internal/agenteval/run.go | 10 +-- internal/dictation/runner.go | 3 +- internal/execution/command_context.go | 55 ++++++++++++++--- internal/execution/command_context_test.go | 61 +++++++++++++++++++ .../execution/command_context_unix_test.go | 19 ++++++ .../execution/command_context_windows_test.go | 36 +++++++++++ internal/execution/command_tree_unix.go | 41 +++++++++++++ internal/execution/command_tree_windows.go | 60 ++++++++++++++++++ internal/hooks/dispatch.go | 3 +- internal/hooks/dispatch_test.go | 40 +++++++++--- internal/hooks/process_test_unix.go | 28 +++++++++ internal/hooks/process_test_windows.go | 36 +++++++++++ internal/perfbench/perfbench.go | 11 ++-- internal/perfbench/taskbench.go | 14 +++-- internal/perfbench/taskbench_test.go | 15 +++++ internal/perfbench/turn_bench.go | 8 ++- internal/perfbench/turn_bench_test.go | 14 +++++ internal/verify/verify.go | 7 ++- internal/verify/verify_test.go | 36 +++++++++++ 21 files changed, 456 insertions(+), 47 deletions(-) create mode 100644 internal/execution/command_context_test.go create mode 100644 internal/execution/command_context_unix_test.go create mode 100644 internal/execution/command_context_windows_test.go create mode 100644 internal/execution/command_tree_unix.go create mode 100644 internal/execution/command_tree_windows.go create mode 100644 internal/hooks/process_test_unix.go create mode 100644 internal/hooks/process_test_windows.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index dc4cf5197..8919529ea 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -63,14 +63,13 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A } command := expandAgentCommand(runner.Command, input) cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = input.WorkspacePath stdout := &capWriter{limit: limit} stderr := &capWriter{limit: limit} cmd.Stdout = stdout cmd.Stderr = stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result.Stdout = stdout.buf.String() result.Stderr = stderr.buf.String() result.Truncated = stdout.truncated || stderr.truncated diff --git a/internal/agenteval/materialize.go b/internal/agenteval/materialize.go index e5076eb3e..3ab705a4a 100644 --- a/internal/agenteval/materialize.go +++ b/internal/agenteval/materialize.go @@ -190,7 +190,6 @@ func initGitBaseline(ctx context.Context, workspace string) error { } for _, args := range commands { cmd := exec.CommandContext(ctx, "git", args...) - execution.HardenCommandContext(cmd) cmd.Dir = workspace cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=Zero Eval", @@ -201,7 +200,7 @@ func initGitBaseline(ctx context.Context, workspace string) error { var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - if err := cmd.Run(); err != nil { + if err := execution.RunCommand(ctx, cmd); err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index 2e8b53557..07a60471a 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -141,13 +141,12 @@ func execCommand(ctx context.Context, workspace string, command Command) Command } args := trimCommand(command.Command) cmd := exec.CommandContext(ctx, args[0], args[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = workspace var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result.Stdout = stdout.String() result.Stderr = stderr.String() if err == nil { @@ -170,15 +169,16 @@ func execCommand(ctx context.Context, workspace string, command Command) Command func defaultRunGit(ctx context.Context, workspace string, args ...string) ([]byte, error) { allArgs := append([]string{"-C", workspace}, args...) cmd := exec.CommandContext(ctx, "git", allArgs...) - execution.HardenCommandContext(cmd) - output, err := cmd.Output() + var output bytes.Buffer + cmd.Stdout = &output + err := execution.RunCommand(ctx, cmd) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return nil, ctxErr } return nil, err } - return output, nil + return output.Bytes(), nil } func parseGitStatusPorcelain(output []byte) []string { diff --git a/internal/dictation/runner.go b/internal/dictation/runner.go index bdfe6362a..43b250559 100644 --- a/internal/dictation/runner.go +++ b/internal/dictation/runner.go @@ -106,11 +106,10 @@ func (p *realProcess) Kill() error { return p.cmd.Process.Kill() } func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) - execution.HardenCommandContext(cmd) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) return out.Bytes(), err } diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 7f21fa71a..6e40af31a 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -1,22 +1,57 @@ package execution import ( + "context" + "errors" + "fmt" "os/exec" ) -// HardenCommandContext makes a context-bound command terminate its process -// tree and prevents inherited output handles from blocking Wait indefinitely. -// Call this before Start or Run. -func HardenCommandContext(command *exec.Cmd) { +// RunCommand runs a context-bound command in a retained process tree and +// prevents inherited output handles from blocking Wait indefinitely. +func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if command == nil { - return + return errors.New("execution: nil command") } - ConfigureProcessGroup(command) + if ctx == nil { + ctx = context.Background() + } + tree, err := prepareCommandTree(command) + if err != nil { + return err + } + defer func() { err = errors.Join(err, tree.close()) }() + command.WaitDelay = processWaitDelay - command.Cancel = func() error { - if command.Process == nil { - return nil + command.Cancel = tree.cancel + if err := command.Start(); err != nil { + _ = tree.attach(nil) + return err + } + if err := tree.attach(command.Process); err != nil { + killErr := command.Process.Kill() + waitErr := command.Wait() + return errors.Join(fmt.Errorf("execution: attach process tree: %w", err), killErr, waitErr) + } + waitComplete := make(chan struct{}) + type cancellation struct { + err error + canceled bool + } + cancelResult := make(chan cancellation, 1) + go func() { + select { + case <-ctx.Done(): + cancelResult <- cancellation{err: tree.cancel(), canceled: true} + case <-waitComplete: + cancelResult <- cancellation{} } - return KillProcessTree(command.Process.Pid) + }() + waitErr := command.Wait() + close(waitComplete) + canceled := <-cancelResult + if canceled.canceled { + return errors.Join(waitErr, ctx.Err(), canceled.err) } + return waitErr } diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go new file mode 100644 index 000000000..7c0c052da --- /dev/null +++ b/internal/execution/command_context_test.go @@ -0,0 +1,61 @@ +package execution + +import ( + "bytes" + "context" + "os" + "os/exec" + "strconv" + "testing" + "time" +) + +func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { + switch os.Getenv("ZERO_COMMAND_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") + child.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") + cmd.Env = append(os.Environ(), + "ZERO_COMMAND_TREE_HELPER=root", + "ZERO_COMMAND_TREE_PID_FILE="+pidFile, + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + + started := time.Now() + err := RunCommand(ctx, cmd) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) + } + if err == nil { + t.Fatal("timed-out command unexpectedly succeeded") + } + pidData, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + } + pid, parseErr := strconv.Atoi(string(pidData)) + if parseErr != nil { + t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + } + awaitProcessExit(t, pid) +} diff --git a/internal/execution/command_context_unix_test.go b/internal/execution/command_context_unix_test.go new file mode 100644 index 000000000..2be93eb9f --- /dev/null +++ b/internal/execution/command_context_unix_test.go @@ -0,0 +1,19 @@ +//go:build !windows + +package execution + +import ( + "testing" + "time" +) + +func awaitProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for signalTargetRunning(pid) { + if time.Now().After(deadline) { + t.Fatalf("descendant process %d is still running after command cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go new file mode 100644 index 000000000..9ad436051 --- /dev/null +++ b/internal/execution/command_context_windows_test.go @@ -0,0 +1,36 @@ +//go:build windows + +package execution + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const processStillActive = 259 + +func awaitProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for processIsActive(pid) { + if time.Now().After(deadline) { + t.Fatalf("descendant process %d is still running after command cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} + +func processIsActive(pid int) bool { + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return false + } + defer windows.CloseHandle(handle) + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + return false + } + return exitCode == processStillActive +} diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go new file mode 100644 index 000000000..a0c3e45ee --- /dev/null +++ b/internal/execution/command_tree_unix.go @@ -0,0 +1,41 @@ +//go:build !windows + +package execution + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +type commandTree struct { + ready chan struct{} + pid int +} + +func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { + ConfigureProcessGroup(command) + return &commandTree{ready: make(chan struct{})}, nil +} + +func (tree *commandTree) attach(process *os.Process) error { + if process != nil { + tree.pid = process.Pid + } + close(tree.ready) + return nil +} + +func (tree *commandTree) cancel() error { + <-tree.ready + if tree.pid <= 1 { + return nil + } + if err := syscall.Kill(-tree.pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} + +func (tree *commandTree) close() error { return tree.cancel() } diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go new file mode 100644 index 000000000..73fa69fbc --- /dev/null +++ b/internal/execution/command_tree_windows.go @@ -0,0 +1,60 @@ +//go:build windows + +package execution + +import ( + "fmt" + "os" + "os/exec" + "unsafe" + + "golang.org/x/sys/windows" +) + +type commandTree struct { + job windows.Handle + ready chan struct{} +} + +func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("execution: create process job: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + _ = windows.CloseHandle(job) + return nil, fmt.Errorf("execution: configure process job: %w", err) + } + return &commandTree{job: job, ready: make(chan struct{})}, nil +} + +func (tree *commandTree) attach(process *os.Process) error { + defer close(tree.ready) + if process == nil { + return nil + } + handle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(process.Pid), + ) + if err != nil { + return err + } + defer windows.CloseHandle(handle) + return windows.AssignProcessToJobObject(tree.job, handle) +} + +func (tree *commandTree) cancel() error { + <-tree.ready + return windows.TerminateJobObject(tree.job, 1) +} + +func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index 946ed4b3a..3b5c998ab 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -298,7 +298,6 @@ func (dispatcher *Dispatcher) recordCompleted(hook Definition, input DispatchInp // ExitCode (not Err); Err is reserved for commands that could not be launched. func execCommandRunner(ctx context.Context, command string, args []string, stdin []byte, cwd string, env []string) commandResult { cmd := exec.CommandContext(ctx, command, args...) - execution.HardenCommandContext(cmd) cmd.Dir = cwd cmd.Env = env if len(stdin) > 0 { @@ -307,7 +306,7 @@ func execCommandRunner(ctx context.Context, command string, args []string, stdin var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(ctx, cmd) result := commandResult{Stdout: stdout.String(), Stderr: stderr.String()} if err == nil { return result diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 099f3b5be..c99e70e2f 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "testing" "time" @@ -36,29 +37,52 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { if err := child.Start(); err != nil { os.Exit(2) } + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } select {} case "grandchild": time.Sleep(30 * time.Second) os.Exit(0) } + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() started := time.Now() - result := execCommandRunner( - ctx, - os.Args[0], - []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, - nil, - "", - append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent"), - ) + resultChannel := make(chan commandResult, 1) + go func() { + resultChannel <- execCommandRunner( + ctx, + os.Args[0], + []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, + nil, + "", + append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent", "ZERO_HOOK_TREE_PID_FILE="+pidFile), + ) + }() + var result commandResult + select { + case result = <-resultChannel: + case <-time.After(4 * time.Second): + cancel() + t.Fatal("execCommandRunner did not return within four seconds after its timeout") + } if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) } if result.Err == nil && result.ExitCode == 0 { t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) } + pidData, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read grandchild PID: %v", err) + } + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("parse grandchild PID %q: %v", pidData, err) + } + awaitHookProcessExit(t, pid) } func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { diff --git a/internal/hooks/process_test_unix.go b/internal/hooks/process_test_unix.go new file mode 100644 index 000000000..01e22f2a8 --- /dev/null +++ b/internal/hooks/process_test_unix.go @@ -0,0 +1,28 @@ +//go:build !windows + +package hooks + +import ( + "errors" + "syscall" + "testing" + "time" +) + +func awaitHookProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + return + } + if err != nil { + t.Fatalf("probe grandchild process %d: %v", pid, err) + } + if time.Now().After(deadline) { + t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/hooks/process_test_windows.go b/internal/hooks/process_test_windows.go new file mode 100644 index 000000000..2b859f545 --- /dev/null +++ b/internal/hooks/process_test_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package hooks + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +const processStillActive = 259 + +func awaitHookProcessExit(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for hookProcessIsActive(pid) { + if time.Now().After(deadline) { + t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) + } + time.Sleep(10 * time.Millisecond) + } +} + +func hookProcessIsActive(pid int) bool { + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return false + } + defer windows.CloseHandle(handle) + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + return false + } + return exitCode == processStillActive +} diff --git a/internal/perfbench/perfbench.go b/internal/perfbench/perfbench.go index 762ee1ee9..97435ff83 100644 --- a/internal/perfbench/perfbench.go +++ b/internal/perfbench/perfbench.go @@ -267,12 +267,14 @@ func MeasureColdStart(ctx context.Context, command []string) (float64, error) { } startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) - output, err := cmd.CombinedOutput() + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + err := execution.RunCommand(ctx, cmd) durationMs := RoundMetric(float64(time.Since(startedAt).Microseconds()) / 1000) if err != nil { - return 0, commandError(command, err, string(output), "") + return 0, commandError(command, err, output.String(), "") } return durationMs, nil } @@ -284,7 +286,6 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl rssBefore := readHarnessMemoryMb() startedAt := time.Now() cmd := exec.CommandContext(ctx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Env = offlineBenchmarkEnv(os.Environ()) var once sync.Once @@ -298,7 +299,7 @@ func MeasureFirstOutput(ctx context.Context, command []string) (firstOutputSampl stderr := &timedBuffer{onFirstWrite: markFirstOutput} cmd.Stdout = stdout cmd.Stderr = stderr - waitErr := cmd.Run() + waitErr := execution.RunCommand(ctx, cmd) finishedAt := time.Now() if firstOutputAt.IsZero() { diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index ea9a0d86f..a922bd998 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -330,7 +330,6 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { return func(ctx context.Context, task BenchTask, rc RunContext) TaskOutcome { args := buildExecArgs(task, rc, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -338,7 +337,10 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - runErr := cmd.Run() + runErr := execution.RunCommand(ctx, cmd) + if errors.Is(runErr, exec.ErrWaitDelay) { + return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + } // The terminal run_end exit code is authoritative for pass/fail: a non-zero // agent exit is a normal task failure, not a harness error, even though @@ -386,7 +388,6 @@ func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { func runVerification(ctx context.Context, task BenchTask) TaskOutcome { cmd := exec.CommandContext(ctx, task.VerificationCommand[0], task.VerificationCommand[1:]...) - execution.HardenCommandContext(cmd) // Inherit the environment so the verifier sees PATH, HOME, language toolchain // vars, etc. — the same surface a maintainer gets running the command by hand // (matching the agent run above). NO_COLOR is appended for stable output. @@ -394,9 +395,12 @@ func runVerification(ctx context.Context, task BenchTask) TaskOutcome { if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir } - output, err := cmd.CombinedOutput() + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + err := execution.RunCommand(ctx, cmd) if err != nil { - detail := strings.TrimSpace(string(output)) + detail := strings.TrimSpace(output.String()) if detail == "" { detail = err.Error() } diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index 4f08d072f..ee55367b5 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -319,6 +319,21 @@ exit 0 } } +func TestNewExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { + stub := writeExecStub(t, `sleep 30 & +echo '{"type":"run_end","exitCode":0}' +exit 0 +`) + runner := NewExecRunner(stub) + outcome := runner(context.Background(), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + if outcome.Err == nil || !strings.Contains(outcome.Err.Error(), "output cleanup failed") { + t.Fatalf("inherited output pipe must be a harness error, got %#v", outcome) + } + if outcome.Passed { + t.Fatal("run_end must not bypass an output cleanup failure") + } +} + func TestNewExecRunnerLaunchFailureIsHarnessError(t *testing.T) { // A binary that cannot be launched (no terminal event, process error) is a // genuine harness error. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 788a95944..1e30db2cf 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -658,7 +658,6 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { args := buildTurnExecArgs(task, rc, tracePath, extraArgs) cmd := exec.CommandContext(ctx, binary, args...) - execution.HardenCommandContext(cmd) cmd.Env = appendNoColor(os.Environ()) if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" { cmd.Dir = dir @@ -667,12 +666,15 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { cmd.Stdout = &outBuf cmd.Stderr = &errBuf start := time.Now() - runErr := cmd.Run() + runErr := execution.RunCommand(ctx, cmd) wallMs := float64(time.Since(start).Microseconds()) / 1000 exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) outcome := TurnTaskOutcome{WallMs: wallMs} - if haveExit && exitCode != 0 { + if errors.Is(runErr, exec.ErrWaitDelay) { + outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + return outcome + } else if haveExit && exitCode != 0 { outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode) } else if !haveExit { detail := strings.TrimSpace(errBuf.String()) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 5b2295654..d91597241 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -643,6 +643,20 @@ func runTurnStub(t *testing.T, task BenchTask, stubBody string) TurnTaskOutcome return NewTurnExecRunner(stub)(context.Background(), task, RunContext{Model: "fake-model"}) } +func TestNewTurnExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { + task := BenchTask{ID: "wait-delay", Prompt: "p", WorkspaceFixture: t.TempDir()} + outcome := runTurnStub(t, task, `sleep 30 & +echo '{"type":"run_end","exitCode":0}' +exit 0 +`) + if outcome.Err == nil || !strings.Contains(outcome.Err.Error(), "output cleanup failed") { + t.Fatalf("inherited output pipe must be a harness error, got %#v", outcome) + } + if outcome.Passed { + t.Fatal("run_end must not bypass an output cleanup failure") + } +} + // assertVerifyFailed asserts an outcome failed specifically because the oracle // rejected the work — Passed is false, there is no harness error (Err nil), and // VerifyErr carries the surfaced failure detail. This is stronger than merely diff --git a/internal/verify/verify.go b/internal/verify/verify.go index cd152bf41..a571cada8 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -3,6 +3,7 @@ package verify import ( "bytes" "context" + "errors" "fmt" "os" "os/exec" @@ -300,17 +301,17 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti defer cancel() cmd := exec.CommandContext(commandCtx, command[0], command[1:]...) - execution.HardenCommandContext(cmd) cmd.Dir = dir var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr - err := cmd.Run() + err := execution.RunCommand(commandCtx, cmd) exitCode := 0 if err != nil { exitCode = -1 - if exitError, ok := err.(*exec.ExitError); ok { + var exitError *exec.ExitError + if errors.As(err, &exitError) { exitCode = exitError.ExitCode() err = nil } diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go index 5a9291614..b8792ca09 100644 --- a/internal/verify/verify_test.go +++ b/internal/verify/verify_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -12,6 +13,41 @@ import ( "github.com/Gitlawb/zero/internal/testrunner" ) +func TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { + switch os.Getenv("ZERO_VERIFY_TREE_HELPER") { + case "parent": + if err := os.Setenv("ZERO_VERIFY_TREE_HELPER", "grandchild"); err != nil { + os.Exit(2) + } + child := exec.Command(os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$") + child.Env = os.Environ() + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(3) + } + select {} + case "grandchild": + time.Sleep(30 * time.Second) + return + } + + t.Setenv("ZERO_VERIFY_TREE_HELPER", "parent") + plan := Plan{Root: t.TempDir(), Checks: []Check{{ + ID: "tree.timeout", + Name: "process tree timeout", + Command: []string{os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$"}, + }}} + started := time.Now() + report := Run(context.Background(), plan, RunOptions{TimeoutMS: 100}) + if elapsed := time.Since(started); elapsed > 2*time.Second { + t.Fatalf("defaultRunner remained blocked by grandchild output handles for %s", elapsed) + } + if report.OK || len(report.Results) != 1 || report.Results[0].Status == StatusPass { + t.Fatalf("timed-out defaultRunner command unexpectedly passed: %#v", report) + } +} + func TestDetectPlanFindsBunAndGoChecks(t *testing.T) { root := t.TempDir() writeFile(t, filepath.Join(root, "go.mod"), "module example.com/zero\n") From 8e69a0b70d889fabfd433d5e98681ce0dfe36dfc Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 27 Aug 2026 21:21:39 +0000 Subject: [PATCH 10/17] fix(process): bind Windows jobs before execution Amp-Thread-ID: https://ampcode.com/threads/T-01a044e2-92ad-774b-9a86-094e2e5293bf Co-authored-by: Pierre Bruno --- internal/execution/command_tree_unix.go | 2 +- internal/execution/command_tree_windows.go | 46 ++++++++++++++++++++-- internal/hooks/dispatch_test.go | 2 +- internal/perfbench/taskbench_test.go | 2 +- internal/perfbench/turn_bench_test.go | 2 +- 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go index a0c3e45ee..eb00dfef1 100644 --- a/internal/execution/command_tree_unix.go +++ b/internal/execution/command_tree_unix.go @@ -38,4 +38,4 @@ func (tree *commandTree) cancel() error { return nil } -func (tree *commandTree) close() error { return tree.cancel() } +func (*commandTree) close() error { return nil } diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index 73fa69fbc..d70e36bbc 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,9 +3,11 @@ package execution import ( + "errors" "fmt" "os" "os/exec" + "syscall" "unsafe" "golang.org/x/sys/windows" @@ -16,7 +18,7 @@ type commandTree struct { ready chan struct{} } -func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { +func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { job, err := windows.CreateJobObject(nil, nil) if err != nil { return nil, fmt.Errorf("execution: create process job: %w", err) @@ -32,6 +34,10 @@ func prepareCommandTree(_ *exec.Cmd) (*commandTree, error) { _ = windows.CloseHandle(job) return nil, fmt.Errorf("execution: configure process job: %w", err) } + if command.SysProcAttr == nil { + command.SysProcAttr = &syscall.SysProcAttr{} + } + command.SysProcAttr.CreationFlags |= windows.CREATE_SUSPENDED return &commandTree{job: job, ready: make(chan struct{})}, nil } @@ -48,8 +54,13 @@ func (tree *commandTree) attach(process *os.Process) error { if err != nil { return err } - defer windows.CloseHandle(handle) - return windows.AssignProcessToJobObject(tree.job, handle) + if err := windows.AssignProcessToJobObject(tree.job, handle); err != nil { + return errors.Join(err, windows.CloseHandle(handle)) + } + if err := windows.CloseHandle(handle); err != nil { + return err + } + return resumeProcess(uint32(process.Pid)) } func (tree *commandTree) cancel() error { @@ -58,3 +69,32 @@ func (tree *commandTree) cancel() error { } func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } + +func resumeProcess(pid uint32) (err error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return err + } + defer func() { err = errors.Join(err, windows.CloseHandle(snapshot)) }() + + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + if err := windows.Thread32First(snapshot, &entry); err != nil { + return err + } + for { + if entry.OwnerProcessID == pid { + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + if err != nil { + return err + } + _, resumeErr := windows.ResumeThread(thread) + return errors.Join(resumeErr, windows.CloseHandle(thread)) + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if errors.Is(err, windows.ERROR_NO_MORE_FILES) { + return fmt.Errorf("execution: no thread found for suspended process %d", pid) + } + return err + } + } +} diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index c99e70e2f..4ce1b9ba7 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -47,7 +47,7 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { } pidFile := filepath.Join(t.TempDir(), "grandchild.pid") - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() started := time.Now() resultChannel := make(chan commandResult, 1) diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index ee55367b5..e42d0af82 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -320,7 +320,7 @@ exit 0 } func TestNewExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { - stub := writeExecStub(t, `sleep 30 & + stub := writeExecStub(t, `sleep 3 & echo '{"type":"run_end","exitCode":0}' exit 0 `) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index d91597241..2be6d1a1f 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -645,7 +645,7 @@ func runTurnStub(t *testing.T, task BenchTask, stubBody string) TurnTaskOutcome func TestNewTurnExecRunnerWaitDelayCannotPassWithRunEnd(t *testing.T) { task := BenchTask{ID: "wait-delay", Prompt: "p", WorkspaceFixture: t.TempDir()} - outcome := runTurnStub(t, task, `sleep 30 & + outcome := runTurnStub(t, task, `sleep 3 & echo '{"type":"run_end","exitCode":0}' exit 0 `) From 20b24c1abf72fbb92573d7eb673934ba9856f14e Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Fri, 28 Aug 2026 20:57:11 +0200 Subject: [PATCH 11/17] fix(execution): preserve successful Windows descendants Co-authored-by: Pierre Bruno --- .../execution/command_context_windows_test.go | 72 +++++++++++++++++++ internal/execution/command_tree_windows.go | 11 --- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go index 9ad436051..bae0f7577 100644 --- a/internal/execution/command_context_windows_test.go +++ b/internal/execution/command_context_windows_test.go @@ -3,6 +3,12 @@ package execution import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" "testing" "time" @@ -34,3 +40,69 @@ func processIsActive(pid int) bool { } return exitCode == processStillActive } + +func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { + switch os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) + } + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") + child.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child") + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + child.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: windows.DETACHED_PROCESS | windows.CREATE_NEW_PROCESS_GROUP, + } + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(4) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx := context.Background() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") + command.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=root", + "ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE="+pidFile, + ) + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand failed: %v", err) + } + pidData, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read detached child PID: %v", err) + } + pid, err := strconv.Atoi(string(pidData)) + if err != nil { + t.Fatalf("parse detached child PID %q: %v", pidData, err) + } + t.Cleanup(func() { + if !processIsActive(pid) { + return + } + process, findErr := os.FindProcess(pid) + if findErr != nil { + t.Errorf("find detached child %d: %v", pid, findErr) + return + } + if killErr := process.Kill(); killErr != nil { + t.Errorf("kill detached child %d: %v", pid, killErr) + return + } + awaitProcessExit(t, pid) + }) + if !processIsActive(pid) { + t.Fatalf("successful RunCommand terminated detached child %d", pid) + } +} diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index d70e36bbc..4fb7ea9d8 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -23,17 +23,6 @@ func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { if err != nil { return nil, fmt.Errorf("execution: create process job: %w", err) } - info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} - info.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - if _, err := windows.SetInformationJobObject( - job, - windows.JobObjectExtendedLimitInformation, - uintptr(unsafe.Pointer(&info)), - uint32(unsafe.Sizeof(info)), - ); err != nil { - _ = windows.CloseHandle(job) - return nil, fmt.Errorf("execution: configure process job: %w", err) - } if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } From c6e35f0bd6a957f21c5409440850504a808be7ac Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sat, 29 Aug 2026 10:38:11 +0200 Subject: [PATCH 12/17] fix(execution): fall back when tree containment fails Amp-Thread-ID: https://ampcode.com/threads/T-01a04c92-2d1d-7508-91bc-416341b7e8b0 Co-authored-by: Amp Co-authored-by: Pierre Bruno --- internal/execution/command_context.go | 3 + internal/execution/command_context_test.go | 50 ++++++++++ internal/execution/command_tree_windows.go | 93 +++++++++++++++---- .../execution/command_tree_windows_test.go | 77 +++++++++++++++ 4 files changed, 206 insertions(+), 17 deletions(-) create mode 100644 internal/execution/command_tree_windows_test.go diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 6e40af31a..015a7e151 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -53,5 +53,8 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if canceled.canceled { return errors.Join(waitErr, ctx.Err(), canceled.err) } + if waitErr != nil { + return errors.Join(waitErr, tree.cancel()) + } return waitErr } diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index 7c0c052da..2af3642ff 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -3,6 +3,7 @@ package execution import ( "bytes" "context" + "errors" "os" "os/exec" "strconv" @@ -59,3 +60,52 @@ func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { } awaitProcessExit(t, pid) } + +func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { + switch os.Getenv("ZERO_WAIT_DELAY_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") + child.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return + } + + pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" + ctx := context.Background() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") + cmd.Env = append(os.Environ(), + "ZERO_WAIT_DELAY_TREE_HELPER=root", + "ZERO_WAIT_DELAY_TREE_PID_FILE="+pidFile, + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + + started := time.Now() + err := RunCommand(ctx, cmd) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) + } + if !errors.Is(err, exec.ErrWaitDelay) { + t.Fatalf("RunCommand error = %v, want exec.ErrWaitDelay", err) + } + pidData, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + } + pid, parseErr := strconv.Atoi(string(pidData)) + if parseErr != nil { + t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + } + awaitProcessExit(t, pid) +} diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index 4fb7ea9d8..dcf2bce2a 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,10 +3,12 @@ package execution import ( + "context" "errors" "fmt" "os" "os/exec" + "strconv" "syscall" "unsafe" @@ -14,15 +16,20 @@ import ( ) type commandTree struct { - job windows.Handle - ready chan struct{} + job windows.Handle + processHandle windows.Handle + process *os.Process + contained bool + ready chan struct{} } +const commandProcessStillActive = uint32(259) // STILL_ACTIVE + func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - job, err := windows.CreateJobObject(nil, nil) - if err != nil { - return nil, fmt.Errorf("execution: create process job: %w", err) - } + // Job containment is preferred but optional. The process still starts + // suspended so attach can retain its identity and either assign the job or + // establish the fallback before any child process can escape. + job, _ := windows.CreateJobObject(nil, nil) if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } @@ -35,29 +42,81 @@ func (tree *commandTree) attach(process *os.Process) error { if process == nil { return nil } + tree.process = process handle, err := windows.OpenProcess( - windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(process.Pid), ) - if err != nil { - return err - } - if err := windows.AssignProcessToJobObject(tree.job, handle); err != nil { - return errors.Join(err, windows.CloseHandle(handle)) - } - if err := windows.CloseHandle(handle); err != nil { - return err + if err == nil { + tree.processHandle = handle + if tree.job != 0 && assignCommandProcessToJob(tree.job, handle) == nil { + tree.contained = true + } + } else { + // PROCESS_SET_QUOTA is needed only for job assignment. If the host + // denies that setup right, retry with the narrower rights needed by the + // retained identity-safe fallback. + tree.processHandle, _ = windows.OpenProcess( + windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, + false, + uint32(process.Pid), + ) } + // Job creation, process opening, and assignment can all fail when the host + // already constrains this process. They must not strand it suspended. return resumeProcess(uint32(process.Pid)) } func (tree *commandTree) cancel() error { <-tree.ready - return windows.TerminateJobObject(tree.job, 1) + if tree.contained { + return windows.TerminateJobObject(tree.job, 1) + } + if tree.processHandle == 0 { + if tree.process == nil { + return nil + } + return tree.process.Kill() + } + + // The retained handle both identifies the original process and prevents its + // PID from being reused. Only ask taskkill to walk by PID while that exact + // process is still active; after Wait has reaped it, numeric PID targeting + // could otherwise hit an unrelated process. + var exitCode uint32 + if err := windows.GetExitCodeProcess(tree.processHandle, &exitCode); err != nil { + return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) + } + if exitCode != commandProcessStillActive { + return nil + } + if err := cancelCommandTreeByPID(tree.process.Pid); err == nil { + return nil + } else { + return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) + } } -func (tree *commandTree) close() error { return windows.CloseHandle(tree.job) } +func (tree *commandTree) close() (err error) { + if tree.job != 0 { + err = errors.Join(err, windows.CloseHandle(tree.job)) + tree.job = 0 + } + if tree.processHandle != 0 { + err = errors.Join(err, windows.CloseHandle(tree.processHandle)) + tree.processHandle = 0 + } + return err +} + +var assignCommandProcessToJob = windows.AssignProcessToJobObject + +var cancelCommandTreeByPID = func(pid int) error { + ctx, cancel := context.WithTimeout(context.Background(), processWaitDelay) + defer cancel() + return exec.CommandContext(ctx, taskkillPath(), "/T", "/F", "/PID", strconv.Itoa(pid)).Run() +} func resumeProcess(pid uint32) (err error) { snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go new file mode 100644 index 000000000..ef9ac9de0 --- /dev/null +++ b/internal/execution/command_tree_windows_test.go @@ -0,0 +1,77 @@ +//go:build windows + +package execution + +import ( + "context" + "os/exec" + "testing" + + "golang.org/x/sys/windows" +) + +func TestRunCommandContinuesWhenJobAssignmentFails(t *testing.T) { + originalAssign := assignCommandProcessToJob + assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + return windows.ERROR_ACCESS_DENIED + } + t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) + + ctx := context.Background() + command := exec.CommandContext(ctx, "cmd", "/C", "exit /b 0") + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand failed after optional job assignment failed: %v", err) + } +} + +func TestCommandTreeFallbackDoesNotTargetExitedProcessPID(t *testing.T) { + originalAssign := assignCommandProcessToJob + assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + return windows.ERROR_ACCESS_DENIED + } + t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) + + taskkillCalls := 0 + originalCancelByPID := cancelCommandTreeByPID + cancelCommandTreeByPID = func(int) error { + taskkillCalls++ + return nil + } + t.Cleanup(func() { cancelCommandTreeByPID = originalCancelByPID }) + + command := exec.Command("cmd", "/C", "exit /b 0") + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepareCommandTree: %v", err) + } + t.Cleanup(func() { + if closeErr := tree.close(); closeErr != nil { + t.Errorf("close command tree: %v", closeErr) + } + }) + if err := command.Start(); err != nil { + _ = tree.attach(nil) + t.Fatalf("Start: %v", err) + } + if err := tree.attach(command.Process); err != nil { + _ = command.Process.Kill() + _ = command.Wait() + t.Fatalf("attachCommandTree: %v", err) + } + if tree.contained { + t.Fatal("command unexpectedly reported job containment after forced assignment failure") + } + if tree.processHandle == 0 { + t.Fatal("command tree did not retain the fallback process identity") + } + if err := command.Wait(); err != nil { + t.Fatalf("Wait: %v", err) + } + + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after root exit: %v", err) + } + if taskkillCalls != 0 { + t.Fatalf("fallback targeted an exited root's numeric PID %d time(s)", taskkillCalls) + } +} From 56a3503046d0f391c144acea2f323df3b2a63051 Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 2 Sep 2026 22:04:33 +0000 Subject: [PATCH 13/17] fix(execution): require Windows job containment Amp-Thread-ID: https://ampcode.com/threads/T-01a06415-151c-75f8-9316-7cf572d58d9b Co-authored-by: Pierre Bruno --- internal/execution/command_tree_windows.go | 70 +++----------- .../execution/command_tree_windows_test.go | 91 ++++++++----------- 2 files changed, 54 insertions(+), 107 deletions(-) diff --git a/internal/execution/command_tree_windows.go b/internal/execution/command_tree_windows.go index dcf2bce2a..db5a64ab3 100644 --- a/internal/execution/command_tree_windows.go +++ b/internal/execution/command_tree_windows.go @@ -3,12 +3,10 @@ package execution import ( - "context" "errors" "fmt" "os" "os/exec" - "strconv" "syscall" "unsafe" @@ -18,18 +16,15 @@ import ( type commandTree struct { job windows.Handle processHandle windows.Handle - process *os.Process contained bool ready chan struct{} } -const commandProcessStillActive = uint32(259) // STILL_ACTIVE - func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - // Job containment is preferred but optional. The process still starts - // suspended so attach can retain its identity and either assign the job or - // establish the fallback before any child process can escape. - job, _ := windows.CreateJobObject(nil, nil) + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return nil, fmt.Errorf("execution: create command job: %w", err) + } if command.SysProcAttr == nil { command.SysProcAttr = &syscall.SysProcAttr{} } @@ -42,29 +37,22 @@ func (tree *commandTree) attach(process *os.Process) error { if process == nil { return nil } - tree.process = process handle, err := windows.OpenProcess( windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(process.Pid), ) - if err == nil { - tree.processHandle = handle - if tree.job != 0 && assignCommandProcessToJob(tree.job, handle) == nil { - tree.contained = true - } - } else { - // PROCESS_SET_QUOTA is needed only for job assignment. If the host - // denies that setup right, retry with the narrower rights needed by the - // retained identity-safe fallback. - tree.processHandle, _ = windows.OpenProcess( - windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, - false, - uint32(process.Pid), - ) + if err != nil { + return fmt.Errorf("open suspended command process: %w", err) + } + tree.processHandle = handle + if err := assignCommandProcessToJob(tree.job, handle); err != nil { + // Without job containment, a root can exit before cancellation and + // leave no identity-safe way to find descendants holding output pipes. + // Fail while it is still suspended so no descendant can escape. + return fmt.Errorf("assign suspended command process to job: %w", err) } - // Job creation, process opening, and assignment can all fail when the host - // already constrains this process. They must not strand it suspended. + tree.contained = true return resumeProcess(uint32(process.Pid)) } @@ -73,29 +61,7 @@ func (tree *commandTree) cancel() error { if tree.contained { return windows.TerminateJobObject(tree.job, 1) } - if tree.processHandle == 0 { - if tree.process == nil { - return nil - } - return tree.process.Kill() - } - - // The retained handle both identifies the original process and prevents its - // PID from being reused. Only ask taskkill to walk by PID while that exact - // process is still active; after Wait has reaped it, numeric PID targeting - // could otherwise hit an unrelated process. - var exitCode uint32 - if err := windows.GetExitCodeProcess(tree.processHandle, &exitCode); err != nil { - return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) - } - if exitCode != commandProcessStillActive { - return nil - } - if err := cancelCommandTreeByPID(tree.process.Pid); err == nil { - return nil - } else { - return errors.Join(err, windows.TerminateProcess(tree.processHandle, 1)) - } + return nil } func (tree *commandTree) close() (err error) { @@ -112,12 +78,6 @@ func (tree *commandTree) close() (err error) { var assignCommandProcessToJob = windows.AssignProcessToJobObject -var cancelCommandTreeByPID = func(pid int) error { - ctx, cancel := context.WithTimeout(context.Background(), processWaitDelay) - defer cancel() - return exec.CommandContext(ctx, taskkillPath(), "/T", "/F", "/PID", strconv.Itoa(pid)).Run() -} - func resumeProcess(pid uint32) (err error) { snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) if err != nil { diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go index ef9ac9de0..06caa260c 100644 --- a/internal/execution/command_tree_windows_test.go +++ b/internal/execution/command_tree_windows_test.go @@ -4,74 +4,61 @@ package execution import ( "context" + "errors" + "os" "os/exec" + "path/filepath" + "strconv" "testing" + "time" "golang.org/x/sys/windows" ) -func TestRunCommandContinuesWhenJobAssignmentFails(t *testing.T) { - originalAssign := assignCommandProcessToJob - assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { - return windows.ERROR_ACCESS_DENIED - } - t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - - ctx := context.Background() - command := exec.CommandContext(ctx, "cmd", "/C", "exit /b 0") - if err := RunCommand(ctx, command); err != nil { - t.Fatalf("RunCommand failed after optional job assignment failed: %v", err) +func TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails(t *testing.T) { + switch os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_HELPER") { + case "root": + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") + child.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child") + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + os.Exit(2) + } + if err := os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + os.Exit(3) + } + return + case "child": + time.Sleep(30 * time.Second) + return } -} -func TestCommandTreeFallbackDoesNotTargetExitedProcessPID(t *testing.T) { originalAssign := assignCommandProcessToJob assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { return windows.ERROR_ACCESS_DENIED } t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - taskkillCalls := 0 - originalCancelByPID := cancelCommandTreeByPID - cancelCommandTreeByPID = func(int) error { - taskkillCalls++ - return nil - } - t.Cleanup(func() { cancelCommandTreeByPID = originalCancelByPID }) - - command := exec.Command("cmd", "/C", "exit /b 0") - tree, err := prepareCommandTree(command) - if err != nil { - t.Fatalf("prepareCommandTree: %v", err) - } - t.Cleanup(func() { - if closeErr := tree.close(); closeErr != nil { - t.Errorf("close command tree: %v", closeErr) - } - }) - if err := command.Start(); err != nil { - _ = tree.attach(nil) - t.Fatalf("Start: %v", err) - } - if err := tree.attach(command.Process); err != nil { - _ = command.Process.Kill() - _ = command.Wait() - t.Fatalf("attachCommandTree: %v", err) - } - if tree.contained { - t.Fatal("command unexpectedly reported job containment after forced assignment failure") - } - if tree.processHandle == 0 { - t.Fatal("command tree did not retain the fallback process identity") + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx := context.Background() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") + command.Env = append(os.Environ(), + "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=root", + "ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE="+pidFile, + ) + started := time.Now() + err := RunCommand(ctx, command) + if elapsed := time.Since(started); elapsed > 4*time.Second { + t.Fatalf("RunCommand took %s after job assignment failed", elapsed) } - if err := command.Wait(); err != nil { - t.Fatalf("Wait: %v", err) + if !errors.Is(err, windows.ERROR_ACCESS_DENIED) { + t.Fatalf("RunCommand error = %v, want ERROR_ACCESS_DENIED", err) } - - if err := tree.cancel(); err != nil { - t.Fatalf("cancel after root exit: %v", err) + if command.ProcessState == nil || !command.ProcessState.Exited() { + t.Fatalf("suspended command was not killed and reaped: state = %v", command.ProcessState) } - if taskkillCalls != 0 { - t.Fatalf("fallback targeted an exited root's numeric PID %d time(s)", taskkillCalls) + if _, statErr := os.Stat(pidFile); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("suspended command spawned a descendant after job assignment failed: PID file error = %v", statErr) } } From 53b3ead8c8ef365ec05b3b67f9c8ac5763ffe237 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Thu, 3 Sep 2026 21:20:29 +0200 Subject: [PATCH 14/17] fix(execution): retain process tree lifecycle ownership Co-authored-by: Pierre Bruno --- internal/agenteval/agent_command.go | 4 +- internal/agenteval/run.go | 3 +- .../command_context_process_unix_test.go | 100 ++++++++++ .../command_context_process_windows_test.go | 174 ++++++++++++++++++ internal/execution/command_context_test.go | 155 ++++++++++++---- .../execution/command_context_unix_test.go | 60 +++++- .../execution/command_context_windows_test.go | 71 ++----- internal/execution/command_tree_unix.go | 79 ++++++-- internal/execution/command_tree_unix_test.go | 174 ++++++++++++++++++ .../execution/command_tree_windows_test.go | 36 ++-- internal/execution/exit_error.go | 31 ++++ internal/execution/exit_error_test.go | 41 +++++ internal/execution/runner.go | 3 +- internal/hooks/dispatch.go | 4 +- internal/hooks/dispatch_test.go | 100 ++++++++-- internal/hooks/process_test_unix.go | 28 --- internal/hooks/process_test_windows.go | 36 ---- internal/hooks/process_unix_test.go | 81 ++++++++ internal/hooks/process_windows_test.go | 95 ++++++++++ internal/perfbench/taskbench.go | 18 +- internal/perfbench/taskbench_test.go | 109 +++++++++++ internal/perfbench/turn_bench.go | 14 +- internal/perfbench/turn_bench_test.go | 53 +++++- internal/verify/process_unix_test.go | 81 ++++++++ internal/verify/process_windows_test.go | 95 ++++++++++ internal/verify/verify.go | 4 +- internal/verify/verify_test.go | 94 +++++++++- 27 files changed, 1512 insertions(+), 231 deletions(-) create mode 100644 internal/execution/command_context_process_unix_test.go create mode 100644 internal/execution/command_context_process_windows_test.go create mode 100644 internal/execution/command_tree_unix_test.go create mode 100644 internal/execution/exit_error.go create mode 100644 internal/execution/exit_error_test.go delete mode 100644 internal/hooks/process_test_unix.go delete mode 100644 internal/hooks/process_test_windows.go create mode 100644 internal/hooks/process_unix_test.go create mode 100644 internal/hooks/process_windows_test.go create mode 100644 internal/verify/process_unix_test.go create mode 100644 internal/verify/process_windows_test.go diff --git a/internal/agenteval/agent_command.go b/internal/agenteval/agent_command.go index 8919529ea..3c2a3f59e 100644 --- a/internal/agenteval/agent_command.go +++ b/internal/agenteval/agent_command.go @@ -3,7 +3,6 @@ package agenteval import ( "bytes" "context" - "errors" "os/exec" "strings" @@ -83,8 +82,7 @@ func (runner CommandAgentRunner) Run(ctx context.Context, input AgentRunInput) A result.Error = ctxErr.Error() return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/agenteval/run.go b/internal/agenteval/run.go index 07a60471a..4e27cffaf 100644 --- a/internal/agenteval/run.go +++ b/internal/agenteval/run.go @@ -153,8 +153,7 @@ func execCommand(ctx context.Context, workspace string, command Command) Command result.ExitCode = 0 return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/execution/command_context_process_unix_test.go b/internal/execution/command_context_process_unix_test.go new file mode 100644 index 000000000..3e23eb4d0 --- /dev/null +++ b/internal/execution/command_context_process_unix_test.go @@ -0,0 +1,100 @@ +//go:build !windows + +package execution + +import ( + "errors" + "os" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +type helperProcessOwner struct { + pidFile string + stopFile string + pid int + exited bool +} + +func ownHelperProcess(t *testing.T, pidFile, stopFile string) *helperProcessOwner { + t.Helper() + owner := &helperProcessOwner{pidFile: pidFile, stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperProcessOwner) waitReady(t *testing.T, timeout time.Duration) int { + t.Helper() + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(owner.pidFile) + if err == nil { + pid, parseErr := strconv.Atoi(strings.TrimSpace(string(data))) + if parseErr == nil && pid > 0 { + owner.pid = pid + return pid + } + } + if time.Now().After(deadline) { + t.Fatalf("helper did not hand off a valid PID within %s", timeout) + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *helperProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request helper process stop: %v", err) + } + if owner.exited { + return + } + if owner.pid == 0 { + data, err := os.ReadFile(owner.pidFile) + if err == nil { + owner.pid, _ = strconv.Atoi(strings.TrimSpace(string(data))) + } + } + if owner.pid <= 0 { + return + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + err := syscall.Kill(owner.pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + owner.pid = 0 + owner.exited = true + return + } + if err != nil { + t.Errorf("check helper process %d after cleanup: %v", owner.pid, err) + return + } + time.Sleep(10 * time.Millisecond) + } + t.Errorf("helper process %d survived cleanup", owner.pid) +} + +func (owner *helperProcessOwner) awaitExit(t *testing.T) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(owner.pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + owner.pid = 0 + owner.exited = true + return + } + if err != nil { + t.Fatalf("check helper process %d: %v", owner.pid, err) + } + if time.Now().After(deadline) { + t.Fatalf("helper process %d is still running after command cancellation", owner.pid) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/execution/command_context_process_windows_test.go b/internal/execution/command_context_process_windows_test.go new file mode 100644 index 000000000..9364062fb --- /dev/null +++ b/internal/execution/command_context_process_windows_test.go @@ -0,0 +1,174 @@ +//go:build windows + +package execution + +import ( + "errors" + "os" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type helperProcessOwner struct { + pidFile string + stopFile string + pid int + handle windows.Handle + exited bool +} + +func ownHelperProcess(t *testing.T, pidFile, stopFile string) *helperProcessOwner { + t.Helper() + owner := &helperProcessOwner{pidFile: pidFile, stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperProcessOwner) waitReady(t *testing.T, timeout time.Duration) int { + t.Helper() + deadline := time.Now().Add(timeout) + for { + observed, err := owner.observeReady() + if err != nil { + t.Fatalf("retain helper process: %v", err) + } + if observed { + return owner.pid + } + if time.Now().After(deadline) { + t.Fatalf("helper did not hand off a valid PID within %s", timeout) + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *helperProcessOwner) observeReady() (bool, error) { + data, err := os.ReadFile(owner.pidFile) + if err != nil { + return false, nil + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return false, nil + } + if owner.handle != 0 { + return true, nil + } + if err := owner.retainPID(pid); err != nil { + return true, err + } + return true, nil +} + +func (owner *helperProcessOwner) retainPID(pid int) error { + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + owner.pid = pid + owner.exited = true + return nil + } + if err != nil { + return err + } + owner.pid = pid + owner.handle = handle + return nil +} + +func (owner *helperProcessOwner) running() bool { + if owner.handle == 0 { + return false + } + var exitCode uint32 + return windows.GetExitCodeProcess(owner.handle, &exitCode) == nil && exitCode == processStillActive +} + +func (owner *helperProcessOwner) awaitExit(t *testing.T) { + t.Helper() + if owner.exited { + return + } + if owner.handle == 0 { + t.Fatal("helper process handle was not retained") + } + status, err := windows.WaitForSingleObject(owner.handle, 2_000) + if err != nil { + t.Fatalf("wait for helper process %d: %v", owner.pid, err) + } + if status != windows.WAIT_OBJECT_0 { + t.Fatalf("helper process %d is still running after command cancellation", owner.pid) + } +} + +func (owner *helperProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request helper process stop: %v", err) + } + if owner.handle == 0 { + observed, err := owner.observeReady() + if observed && err != nil { + t.Errorf("retain helper process for cleanup: %v", err) + } + } + if owner.handle == 0 { + return + } + if owner.running() { + status, err := windows.WaitForSingleObject(owner.handle, 2_000) + if err != nil { + t.Errorf("wait for helper process %d cooperative stop: %v", owner.pid, err) + } else if status == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(owner.handle, 1); err != nil { + t.Errorf("kill helper process %d: %v", owner.pid, err) + } + _, _ = windows.WaitForSingleObject(owner.handle, 2_000) + } + } + if err := windows.CloseHandle(owner.handle); err != nil { + t.Errorf("close helper process %d handle: %v", owner.pid, err) + } + owner.handle = 0 +} + +type helperHandleOwner struct { + handle windows.Handle + stopFile string +} + +func ownHelperHandle(t *testing.T, stopFile string) *helperHandleOwner { + t.Helper() + owner := &helperHandleOwner{stopFile: stopFile} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *helperHandleOwner) retain(process windows.Handle) error { + current := windows.CurrentProcess() + return windows.DuplicateHandle(current, process, current, &owner.handle, windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, 0) +} + +func (owner *helperHandleOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request suspended helper process stop: %v", err) + } + if owner.handle == 0 { + return + } + var exitCode uint32 + if windows.GetExitCodeProcess(owner.handle, &exitCode) == nil && exitCode == processStillActive { + if err := windows.TerminateProcess(owner.handle, 1); err != nil { + t.Errorf("kill suspended helper process: %v", err) + } + _, _ = windows.WaitForSingleObject(owner.handle, 2_000) + } + if err := windows.CloseHandle(owner.handle); err != nil { + t.Errorf("close suspended helper process handle: %v", err) + } + owner.handle = 0 +} diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index 2af3642ff..73cf8fd69 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -15,97 +15,182 @@ func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { switch os.Getenv("ZERO_COMMAND_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") - child.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_COMMAND_TREE_HELPER=child", + "ZERO_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_COMMAND_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterRootExit$") cmd.Env = append(os.Environ(), "ZERO_COMMAND_TREE_HELPER=root", "ZERO_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_COMMAND_TREE_STOP_FILE="+stopFile, ) var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - started := time.Now() - err := RunCommand(ctx, cmd) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) - } + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + cancel() + err := waitForRunCommand(t, result, 4*time.Second) if err == nil { t.Fatal("timed-out command unexpectedly succeeded") } - pidData, readErr := os.ReadFile(pidFile) - if readErr != nil { - t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) - } - pid, parseErr := strconv.Atoi(string(pidData)) - if parseErr != nil { - t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) - } - awaitProcessExit(t, pid) + child.awaitExit(t) } func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { switch os.Getenv("ZERO_WAIT_DELAY_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") - child.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_WAIT_DELAY_TREE_HELPER=child", + "ZERO_WAIT_DELAY_TREE_STOP_FILE="+os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := t.TempDir() + string(os.PathSeparator) + "child.pid" - ctx := context.Background() + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") cmd.Env = append(os.Environ(), "ZERO_WAIT_DELAY_TREE_HELPER=root", "ZERO_WAIT_DELAY_TREE_PID_FILE="+pidFile, + "ZERO_WAIT_DELAY_TREE_STOP_FILE="+stopFile, ) var output bytes.Buffer cmd.Stdout = &output cmd.Stderr = &output - started := time.Now() - err := RunCommand(ctx, cmd) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand remained blocked by descendant output handles for %s", elapsed) - } + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + err := waitForRunCommand(t, result, 4*time.Second) if !errors.Is(err, exec.ErrWaitDelay) { t.Fatalf("RunCommand error = %v, want exec.ErrWaitDelay", err) } - pidData, readErr := os.ReadFile(pidFile) - if readErr != nil { - t.Fatalf("read descendant PID: %v; command output: %s", readErr, output.String()) + child.awaitExit(t) +} + +func TestRunCommandKillsDescendantAfterNonzeroRootExit(t *testing.T) { + switch os.Getenv("ZERO_NONZERO_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) + } + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterNonzeroRootExit$") + child.Env = append(os.Environ(), + "ZERO_NONZERO_TREE_HELPER=child", + "ZERO_NONZERO_TREE_STOP_FILE="+os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), + ) + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_NONZERO_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + // Leave time for the parent test to retain an independent cleanup handle + // before the root's abnormal exit triggers production tree cleanup. + if waitForCommandTreeStop(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), 500*time.Millisecond) { + _ = child.Wait() + return + } + os.Exit(7) + case "child": + waitForCommandTreeStop(os.Getenv("ZERO_NONZERO_TREE_STOP_FILE"), 30*time.Second) + return + } + + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + ctx := context.Background() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandKillsDescendantAfterNonzeroRootExit$") + cmd.Env = append(os.Environ(), + "ZERO_NONZERO_TREE_HELPER=root", + "ZERO_NONZERO_TREE_PID_FILE="+pidFile, + "ZERO_NONZERO_TREE_STOP_FILE="+stopFile, + ) + result := runCommandAsync(ctx, cmd) + child.waitReady(t, 2*time.Second) + err := waitForRunCommand(t, result, 4*time.Second) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 7 { + t.Fatalf("RunCommand error = %v, want exit code 7", err) } - pid, parseErr := strconv.Atoi(string(pidData)) - if parseErr != nil { - t.Fatalf("parse descendant PID %q: %v", pidData, parseErr) + child.awaitExit(t) +} + +func waitForCommandTreeStop(stopFile string, lifetime time.Duration) bool { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + +func runCommandAsync(ctx context.Context, command *exec.Cmd) <-chan error { + result := make(chan error, 1) + go func() { result <- RunCommand(ctx, command) }() + return result +} + +func waitForRunCommand(t *testing.T, result <-chan error, timeout time.Duration) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(timeout): + t.Fatalf("RunCommand did not return within %s", timeout) + return nil } - awaitProcessExit(t, pid) } diff --git a/internal/execution/command_context_unix_test.go b/internal/execution/command_context_unix_test.go index 2be93eb9f..f74cb5e5c 100644 --- a/internal/execution/command_context_unix_test.go +++ b/internal/execution/command_context_unix_test.go @@ -3,17 +3,63 @@ package execution import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" "testing" "time" ) -func awaitProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for signalTargetRunning(pid) { - if time.Now().After(deadline) { - t.Fatalf("descendant process %d is still running after command cancellation", pid) +func TestRunCommandPreservesRedirectedChildAfterSuccessfulExit(t *testing.T) { + switch os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER") { + case "root": + nullFile, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + os.Exit(2) } - time.Sleep(10 * time.Millisecond) + defer nullFile.Close() + child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesRedirectedChildAfterSuccessfulExit$") + child.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER=child", + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), + ) + child.Stdin = nullFile + child.Stdout = nullFile + child.Stderr = nullFile + if err := child.Start(); err != nil { + os.Exit(3) + } + if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + return + case "child": + waitForCommandTreeStop(os.Getenv("ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE"), 30*time.Second) + return + } + + root := t.TempDir() + pidFile := filepath.Join(root, "child.pid") + stopFile := filepath.Join(root, "stop") + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesRedirectedChildAfterSuccessfulExit$") + command.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_HELPER=root", + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_SUCCESSFUL_UNIX_COMMAND_TREE_STOP_FILE="+stopFile, + ) + result := runCommandAsync(ctx, command) + pid := child.waitReady(t, 2*time.Second) + if err := waitForRunCommand(t, result, 4*time.Second); err != nil { + t.Fatalf("RunCommand failed: %v", err) + } + if !signalTargetRunning(pid) { + t.Fatalf("successful RunCommand terminated redirected child %d", pid) } } diff --git a/internal/execution/command_context_windows_test.go b/internal/execution/command_context_windows_test.go index bae0f7577..fbe862039 100644 --- a/internal/execution/command_context_windows_test.go +++ b/internal/execution/command_context_windows_test.go @@ -17,30 +17,6 @@ import ( const processStillActive = 259 -func awaitProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for processIsActive(pid) { - if time.Now().After(deadline) { - t.Fatalf("descendant process %d is still running after command cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} - -func processIsActive(pid int) bool { - handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) - if err != nil { - return false - } - defer windows.CloseHandle(handle) - var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { - return false - } - return exitCode == processStillActive -} - func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { switch os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_HELPER") { case "root": @@ -50,7 +26,10 @@ func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { } defer nullFile.Close() child := exec.Command(os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") - child.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=child", + "ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE="+os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), + ) child.Stdin = nullFile child.Stdout = nullFile child.Stderr = nullFile @@ -61,48 +40,34 @@ func TestRunCommandPreservesDetachedChildAfterSuccessfulExit(t *testing.T) { os.Exit(3) } if err := os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(4) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE"), 30*time.Second) return } - pidFile := filepath.Join(t.TempDir(), "child.pid") - ctx := context.Background() + root := t.TempDir() + pidFile := filepath.Join(root, "child.pid") + stopFile := filepath.Join(root, "stop") + child := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandPreservesDetachedChildAfterSuccessfulExit$") command.Env = append(os.Environ(), "ZERO_SUCCESSFUL_COMMAND_TREE_HELPER=root", "ZERO_SUCCESSFUL_COMMAND_TREE_PID_FILE="+pidFile, + "ZERO_SUCCESSFUL_COMMAND_TREE_STOP_FILE="+stopFile, ) - if err := RunCommand(ctx, command); err != nil { + result := runCommandAsync(ctx, command) + pid := child.waitReady(t, 2*time.Second) + if err := waitForRunCommand(t, result, 4*time.Second); err != nil { t.Fatalf("RunCommand failed: %v", err) } - pidData, err := os.ReadFile(pidFile) - if err != nil { - t.Fatalf("read detached child PID: %v", err) - } - pid, err := strconv.Atoi(string(pidData)) - if err != nil { - t.Fatalf("parse detached child PID %q: %v", pidData, err) - } - t.Cleanup(func() { - if !processIsActive(pid) { - return - } - process, findErr := os.FindProcess(pid) - if findErr != nil { - t.Errorf("find detached child %d: %v", pid, findErr) - return - } - if killErr := process.Kill(); killErr != nil { - t.Errorf("kill detached child %d: %v", pid, killErr) - return - } - awaitProcessExit(t, pid) - }) - if !processIsActive(pid) { + if !child.running() { t.Fatalf("successful RunCommand terminated detached child %d", pid) } } diff --git a/internal/execution/command_tree_unix.go b/internal/execution/command_tree_unix.go index eb00dfef1..02689bed6 100644 --- a/internal/execution/command_tree_unix.go +++ b/internal/execution/command_tree_unix.go @@ -4,38 +4,89 @@ package execution import ( "errors" + "io" "os" "os/exec" + "sync" "syscall" ) type commandTree struct { - ready chan struct{} - pid int + mu sync.Mutex + ready chan struct{} + readyOnce sync.Once + groupID int + anchor *exec.Cmd + anchorInput io.WriteCloser + signal func(int, syscall.Signal) error + canceled bool + cancelErr error + closed bool } func prepareCommandTree(command *exec.Cmd) (*commandTree, error) { - ConfigureProcessGroup(command) - return &commandTree{ready: make(chan struct{})}, nil -} + // Keep the group leader alive until cleanup so an exited command cannot + // leave a reusable PID as the only identity for its live descendants. + anchor := exec.Command("/bin/sh", "-c", "read _") + anchor.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + anchorInput, err := anchor.StdinPipe() + if err != nil { + return nil, err + } + if err := anchor.Start(); err != nil { + _ = anchorInput.Close() + return nil, err + } -func (tree *commandTree) attach(process *os.Process) error { - if process != nil { - tree.pid = process.Pid + tree := &commandTree{ + ready: make(chan struct{}), + groupID: anchor.Process.Pid, + anchor: anchor, + anchorInput: anchorInput, + signal: syscall.Kill, } - close(tree.ready) + if command.SysProcAttr == nil { + command.SysProcAttr = &syscall.SysProcAttr{} + } + command.SysProcAttr.Setpgid = true + command.SysProcAttr.Pgid = tree.groupID + return tree, nil +} + +func (tree *commandTree) attach(*os.Process) error { + tree.readyOnce.Do(func() { close(tree.ready) }) return nil } func (tree *commandTree) cancel() error { <-tree.ready - if tree.pid <= 1 { + tree.mu.Lock() + defer tree.mu.Unlock() + if tree.closed || tree.canceled || tree.groupID <= 1 { + return tree.cancelErr + } + tree.canceled = true + if err := tree.signal(-tree.groupID, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + tree.cancelErr = err + } + return tree.cancelErr +} + +func (tree *commandTree) close() error { + tree.mu.Lock() + defer tree.mu.Unlock() + if tree.closed { return nil } - if err := syscall.Kill(-tree.pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { - return err + tree.closed = true + if tree.anchorInput != nil { + _ = tree.anchorInput.Close() + tree.anchorInput = nil + } + if tree.anchor != nil { + _ = tree.anchor.Wait() + tree.anchor = nil } + tree.groupID = 0 return nil } - -func (*commandTree) close() error { return nil } diff --git a/internal/execution/command_tree_unix_test.go b/internal/execution/command_tree_unix_test.go new file mode 100644 index 000000000..289ed70c5 --- /dev/null +++ b/internal/execution/command_tree_unix_test.go @@ -0,0 +1,174 @@ +//go:build !windows + +package execution + +import ( + "context" + "errors" + "os/exec" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +func TestRunCommandAbsolutePathWithEmptyPATH(t *testing.T) { + t.Setenv("PATH", "") + ctx := context.Background() + command := exec.CommandContext(ctx, "/bin/sh", "-c", "exit 0") + if err := RunCommand(ctx, command); err != nil { + t.Fatalf("RunCommand with absolute executable and empty PATH: %v", err) + } +} + +func TestPrepareCommandTreeRetainsGroupIdentity(t *testing.T) { + attributes := &syscall.SysProcAttr{Setsid: true} + command := exec.Command("sh", "-c", "exit 7") + command.SysProcAttr = attributes + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepare command tree: %v", err) + } + defer tree.close() + + if command.SysProcAttr != attributes { + t.Fatal("prepareCommandTree replaced existing SysProcAttr") + } + if !attributes.Setsid || !attributes.Setpgid || attributes.Pgid != tree.groupID { + t.Fatalf("command attributes = %#v, want preserved Setsid and group %d", attributes, tree.groupID) + } + if pgid, err := syscall.Getpgid(tree.anchor.Process.Pid); err != nil || pgid != tree.groupID { + t.Fatalf("anchor process group = %d, %v; want %d", pgid, err, tree.groupID) + } + + // Setsid and joining an existing process group are intentionally incompatible; + // it is retained above only to verify that unrelated caller fields survive. + attributes.Setsid = false + if err := command.Start(); err != nil { + t.Fatalf("start command: %v", err) + } + if err := tree.attach(command.Process); err != nil { + t.Fatalf("attach command: %v", err) + } + if pgid, err := syscall.Getpgid(command.Process.Pid); err != nil || pgid != tree.groupID { + t.Fatalf("command process group = %d, %v; want %d", pgid, err, tree.groupID) + } + if err := command.Wait(); err == nil { + t.Fatal("command unexpectedly succeeded") + } + if err := syscall.Kill(tree.anchor.Process.Pid, 0); err != nil { + t.Fatalf("anchor did not retain group identity after command exit: %v", err) + } +} + +func TestCommandTreeCancelSignalsOnce(t *testing.T) { + ready := make(chan struct{}) + close(ready) + wantErr := errors.New("signal failed") + var calls atomic.Int32 + tree := &commandTree{ + ready: ready, + groupID: 123, + signal: func(pid int, signal syscall.Signal) error { + calls.Add(1) + if pid != -123 || signal != syscall.SIGKILL { + t.Errorf("signal target = (%d, %v), want (-123, SIGKILL)", pid, signal) + } + return wantErr + }, + } + + const callers = 32 + var wait sync.WaitGroup + wait.Add(callers) + for range callers { + go func() { + defer wait.Done() + if err := tree.cancel(); !errors.Is(err, wantErr) { + t.Errorf("cancel error = %v, want %v", err, wantErr) + } + }() + } + wait.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("signal calls = %d, want 1", got) + } +} + +func TestCommandTreeCloseWaitsForCancelAndPreventsLaterSignals(t *testing.T) { + command := exec.Command("sh", "-c", "exit 0") + tree, err := prepareCommandTree(command) + if err != nil { + t.Fatalf("prepare command tree: %v", err) + } + if err := tree.attach(nil); err != nil { + t.Fatalf("attach command tree: %v", err) + } + anchorPID := tree.anchor.Process.Pid + + signalStarted := make(chan struct{}) + releaseSignal := make(chan struct{}) + var calls atomic.Int32 + tree.signal = func(int, syscall.Signal) error { + calls.Add(1) + close(signalStarted) + <-releaseSignal + return nil + } + cancelDone := make(chan error, 1) + go func() { cancelDone <- tree.cancel() }() + <-signalStarted + + closeDone := make(chan error, 1) + go func() { closeDone <- tree.close() }() + select { + case err := <-closeDone: + t.Fatalf("close returned while signal was in flight: %v", err) + case <-time.After(100 * time.Millisecond): + } + if err := syscall.Kill(anchorPID, 0); err != nil { + t.Fatalf("anchor was released while signal was in flight: %v", err) + } + + close(releaseSignal) + if err := <-cancelDone; err != nil { + t.Fatalf("cancel command tree: %v", err) + } + if err := <-closeDone; err != nil { + t.Fatalf("close command tree: %v", err) + } + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after close: %v", err) + } + if err := tree.close(); err != nil { + t.Fatalf("repeated close: %v", err) + } + if got := calls.Load(); got != 1 { + t.Fatalf("signal calls after close = %d, want 1", got) + } +} + +func TestCommandTreeCancelAfterCloseDoesNotSignal(t *testing.T) { + ready := make(chan struct{}) + close(ready) + var calls atomic.Int32 + tree := &commandTree{ + ready: ready, + groupID: 123, + signal: func(int, syscall.Signal) error { + calls.Add(1) + return nil + }, + } + + if err := tree.close(); err != nil { + t.Fatalf("close command tree: %v", err) + } + if err := tree.cancel(); err != nil { + t.Fatalf("cancel after close: %v", err) + } + if got := calls.Load(); got != 0 { + t.Fatalf("signal calls after close = %d, want 0", got) + } +} diff --git a/internal/execution/command_tree_windows_test.go b/internal/execution/command_tree_windows_test.go index 06caa260c..5c6969f20 100644 --- a/internal/execution/command_tree_windows_test.go +++ b/internal/execution/command_tree_windows_test.go @@ -19,46 +19,58 @@ func TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails(t *testi switch os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_HELPER") { case "root": child := exec.Command(os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") - child.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child") + child.Env = append(os.Environ(), + "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=child", + "ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE="+os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(2) } if err := os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() os.Exit(3) } return case "child": - time.Sleep(30 * time.Second) + waitForCommandTreeStop(os.Getenv("ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE"), 30*time.Second) return } + root := t.TempDir() + stopFile := filepath.Join(root, "stop") originalAssign := assignCommandProcessToJob - assignCommandProcessToJob = func(windows.Handle, windows.Handle) error { + commandOwner := ownHelperHandle(t, stopFile) + assignCommandProcessToJob = func(_ windows.Handle, process windows.Handle) error { + if err := commandOwner.retain(process); err != nil { + return err + } return windows.ERROR_ACCESS_DENIED } t.Cleanup(func() { assignCommandProcessToJob = originalAssign }) - pidFile := filepath.Join(t.TempDir(), "child.pid") - ctx := context.Background() + pidFile := filepath.Join(root, "child.pid") + escapedChild := ownHelperProcess(t, pidFile, stopFile) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunCommandFailsBeforeDescendantCanEscapeWhenJobAssignmentFails$") command.Env = append(os.Environ(), "ZERO_ASSIGNMENT_FAILURE_TREE_HELPER=root", "ZERO_ASSIGNMENT_FAILURE_TREE_PID_FILE="+pidFile, + "ZERO_ASSIGNMENT_FAILURE_TREE_STOP_FILE="+stopFile, ) - started := time.Now() - err := RunCommand(ctx, command) - if elapsed := time.Since(started); elapsed > 4*time.Second { - t.Fatalf("RunCommand took %s after job assignment failed", elapsed) - } + err := waitForRunCommand(t, runCommandAsync(ctx, command), 4*time.Second) if !errors.Is(err, windows.ERROR_ACCESS_DENIED) { t.Fatalf("RunCommand error = %v, want ERROR_ACCESS_DENIED", err) } if command.ProcessState == nil || !command.ProcessState.Exited() { t.Fatalf("suspended command was not killed and reaped: state = %v", command.ProcessState) } - if _, statErr := os.Stat(pidFile); !errors.Is(statErr, os.ErrNotExist) { - t.Fatalf("suspended command spawned a descendant after job assignment failed: PID file error = %v", statErr) + observed, observeErr := escapedChild.observeReady() + _, statErr := os.Stat(pidFile) + if observeErr != nil || observed || !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("suspended command spawned a descendant after job assignment failed: observed = %t, observation error = %v, PID file error = %v", observed, observeErr, statErr) } } diff --git a/internal/execution/exit_error.go b/internal/execution/exit_error.go new file mode 100644 index 000000000..eab627889 --- /dev/null +++ b/internal/execution/exit_error.go @@ -0,0 +1,31 @@ +package execution + +import "os/exec" + +// AsPureExitError reports whether err is an ordinary process exit or a join +// tree containing only ordinary process exits. It does not unwrap single-error +// wrappers, which may carry a distinct lifecycle or cleanup failure. +func AsPureExitError(err error) (*exec.ExitError, bool) { + if exitErr, ok := err.(*exec.ExitError); ok { + return exitErr, exitErr != nil + } + joined, ok := err.(interface{ Unwrap() []error }) + if !ok { + return nil, false + } + causes := joined.Unwrap() + if len(causes) == 0 { + return nil, false + } + var first *exec.ExitError + for _, cause := range causes { + exitErr, ok := AsPureExitError(cause) + if !ok { + return nil, false + } + if first == nil { + first = exitErr + } + } + return first, first != nil +} diff --git a/internal/execution/exit_error_test.go b/internal/execution/exit_error_test.go new file mode 100644 index 000000000..323a87ad7 --- /dev/null +++ b/internal/execution/exit_error_test.go @@ -0,0 +1,41 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "os/exec" + "testing" +) + +func TestAsPureExitError(t *testing.T) { + first := &exec.ExitError{} + second := &exec.ExitError{} + var nilExit *exec.ExitError + tests := []struct { + name string + err error + want *exec.ExitError + ok bool + }{ + {name: "nil"}, + {name: "direct", err: first, want: first, ok: true}, + {name: "joined", err: errors.Join(first, second), want: first, ok: true}, + {name: "nested joins", err: errors.Join(errors.Join(first, second), &exec.ExitError{}), want: first, ok: true}, + {name: "join with nil", err: errors.Join(first, nil), want: first, ok: true}, + {name: "ordinary error", err: errors.New("start failed")}, + {name: "mixed join", err: errors.Join(first, context.Canceled)}, + {name: "nested mixed join", err: errors.Join(first, errors.Join(second, context.DeadlineExceeded))}, + {name: "wrapped exit", err: fmt.Errorf("cleanup failed: %w", first)}, + {name: "join containing wrapped exit", err: errors.Join(first, fmt.Errorf("wrapped: %w", second))}, + {name: "typed nil exit", err: nilExit}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := AsPureExitError(test.err) + if got != test.want || ok != test.ok { + t.Fatalf("AsPureExitError(%v) = (%p, %v), want (%p, %v)", test.err, got, ok, test.want, test.ok) + } + }) + } +} diff --git a/internal/execution/runner.go b/internal/execution/runner.go index 9e3ecbf9a..7da175419 100644 --- a/internal/execution/runner.go +++ b/internal/execution/runner.go @@ -174,8 +174,7 @@ func commandExitCode(err error) int { if err == nil { return 0 } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := AsPureExitError(err); ok { return exitErr.ExitCode() } return -1 diff --git a/internal/hooks/dispatch.go b/internal/hooks/dispatch.go index 3b5c998ab..f7bf6e6de 100644 --- a/internal/hooks/dispatch.go +++ b/internal/hooks/dispatch.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "os" "os/exec" "strings" @@ -311,8 +310,7 @@ func execCommandRunner(ctx context.Context, command string, args []string, stdin if err == nil { return result } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { + if exitErr, ok := execution.AsPureExitError(err); ok { result.ExitCode = exitErr.ExitCode() return result } diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 4ce1b9ba7..7e4de503b 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -30,26 +30,45 @@ func beforeToolConfig(hooks ...Definition) Config { func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { switch os.Getenv("ZERO_HOOK_TREE_HELPER") { case "parent": + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PARENT_PID_FILE"), []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { + os.Exit(2) + } child := exec.Command(os.Args[0], "-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$") - child.Env = append(os.Environ(), "ZERO_HOOK_TREE_HELPER=grandchild") + child.Env = append(os.Environ(), + "ZERO_HOOK_TREE_HELPER=grandchild", + "ZERO_HOOK_TREE_STOP_FILE="+os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { - os.Exit(2) - } - if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { os.Exit(3) } - select {} + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_GRANDCHILD_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + if err := os.WriteFile(os.Getenv("ZERO_HOOK_TREE_READY_FILE"), []byte("ready"), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(5) + } + waitForHookTreeStop(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), 30*time.Second) + _ = child.Wait() + return case "grandchild": - time.Sleep(30 * time.Second) + waitForHookTreeStop(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), 30*time.Second) os.Exit(0) } - pidFile := filepath.Join(t.TempDir(), "grandchild.pid") - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + root := t.TempDir() + parentPIDFile := filepath.Join(root, "parent.pid") + grandchildPIDFile := filepath.Join(root, "grandchild.pid") + readyFile := filepath.Join(root, "ready") + stopFile := filepath.Join(root, "stop") + owner := newHookTestProcessOwner(t, stopFile, parentPIDFile, grandchildPIDFile) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - started := time.Now() resultChannel := make(chan commandResult, 1) go func() { resultChannel <- execCommandRunner( @@ -58,15 +77,23 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}, nil, "", - append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent", "ZERO_HOOK_TREE_PID_FILE="+pidFile), + append(os.Environ(), + "ZERO_HOOK_TREE_HELPER=parent", + "ZERO_HOOK_TREE_PARENT_PID_FILE="+parentPIDFile, + "ZERO_HOOK_TREE_GRANDCHILD_PID_FILE="+grandchildPIDFile, + "ZERO_HOOK_TREE_READY_FILE="+readyFile, + "ZERO_HOOK_TREE_STOP_FILE="+stopFile, + ), ) }() + parentPID, grandchildPID := awaitHookTreeReady(t, owner, readyFile, parentPIDFile, grandchildPIDFile) + started := time.Now() var result commandResult select { case result = <-resultChannel: - case <-time.After(4 * time.Second): + case <-time.After(6 * time.Second): cancel() - t.Fatal("execCommandRunner did not return within four seconds after its timeout") + t.Fatal("execCommandRunner did not return within six seconds after its timeout") } if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("command remained blocked by grandchild output handles for %s", elapsed) @@ -74,15 +101,50 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { if result.Err == nil && result.ExitCode == 0 { t.Fatalf("timed-out command unexpectedly succeeded: %#v", result) } - pidData, err := os.ReadFile(pidFile) - if err != nil { - t.Fatalf("read grandchild PID: %v", err) + for role, pid := range map[string]int{"parent": parentPID, "grandchild": grandchildPID} { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Fatalf("%s process %d survived hook cancellation: %v", role, pid, err) + } } - pid, err := strconv.Atoi(string(pidData)) - if err != nil { - t.Fatalf("parse grandchild PID %q: %v", pidData, err) +} + +func waitForHookTreeStop(stopFile string, lifetime time.Duration) { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } +} + +func awaitHookTreeReady(t *testing.T, owner *hookTestProcessOwner, readyFile string, pidFiles ...string) (int, int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(readyFile); err == nil { + pids := make([]int, 0, len(pidFiles)) + for _, path := range pidFiles { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read helper PID handoff %q: %v", path, err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatalf("parse helper PID handoff %q: %v", data, err) + } + if err := owner.retain(pid); err != nil { + t.Fatalf("retain helper process %d: %v", pid, err) + } + pids = append(pids, pid) + } + return pids[0], pids[1] + } + if time.Now().After(deadline) { + t.Fatal("process-tree helper did not hand off parent and grandchild identities") + } + time.Sleep(10 * time.Millisecond) } - awaitHookProcessExit(t, pid) } func TestDispatchRunsMatchingHooksAndRecordsAudit(t *testing.T) { diff --git a/internal/hooks/process_test_unix.go b/internal/hooks/process_test_unix.go deleted file mode 100644 index 01e22f2a8..000000000 --- a/internal/hooks/process_test_unix.go +++ /dev/null @@ -1,28 +0,0 @@ -//go:build !windows - -package hooks - -import ( - "errors" - "syscall" - "testing" - "time" -) - -func awaitHookProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for { - err := syscall.Kill(pid, syscall.Signal(0)) - if errors.Is(err, syscall.ESRCH) { - return - } - if err != nil { - t.Fatalf("probe grandchild process %d: %v", pid, err) - } - if time.Now().After(deadline) { - t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} diff --git a/internal/hooks/process_test_windows.go b/internal/hooks/process_test_windows.go deleted file mode 100644 index 2b859f545..000000000 --- a/internal/hooks/process_test_windows.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build windows - -package hooks - -import ( - "testing" - "time" - - "golang.org/x/sys/windows" -) - -const processStillActive = 259 - -func awaitHookProcessExit(t *testing.T, pid int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for hookProcessIsActive(pid) { - if time.Now().After(deadline) { - t.Fatalf("grandchild process %d is still alive after hook cancellation", pid) - } - time.Sleep(10 * time.Millisecond) - } -} - -func hookProcessIsActive(pid int) bool { - handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) - if err != nil { - return false - } - defer windows.CloseHandle(handle) - var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { - return false - } - return exitCode == processStillActive -} diff --git a/internal/hooks/process_unix_test.go b/internal/hooks/process_unix_test.go new file mode 100644 index 000000000..af6e6642f --- /dev/null +++ b/internal/hooks/process_unix_test.go @@ -0,0 +1,81 @@ +//go:build !windows + +package hooks + +import ( + "errors" + "os" + "strconv" + "syscall" + "testing" + "time" +) + +type hookTestProcessOwner struct { + pids map[int]struct{} + exited map[int]struct{} + stopFile string + pidFiles []string +} + +func newHookTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *hookTestProcessOwner { + t.Helper() + owner := &hookTestProcessOwner{pids: make(map[int]struct{}), exited: make(map[int]struct{}), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *hookTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + owner.pids[pid] = struct{}{} + return nil +} + +func (owner *hookTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + delete(owner.pids, pid) + owner.exited[pid] = struct{}{} + return nil + } + if err != nil { + return err + } + if time.Now().After(deadline) { + return errors.New("process is still running") + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *hookTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid := range owner.pids { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + } +} + +func (owner *hookTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil && pid > 0 && pid != os.Getpid() { + if _, exited := owner.exited[pid]; !exited { + owner.pids[pid] = struct{}{} + } + } + } +} diff --git a/internal/hooks/process_windows_test.go b/internal/hooks/process_windows_test.go new file mode 100644 index 000000000..06796c923 --- /dev/null +++ b/internal/hooks/process_windows_test.go @@ -0,0 +1,95 @@ +//go:build windows + +package hooks + +import ( + "errors" + "fmt" + "os" + "strconv" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type hookTestProcessOwner struct { + handles map[int]windows.Handle + stopFile string + pidFiles []string +} + +func newHookTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *hookTestProcessOwner { + t.Helper() + owner := &hookTestProcessOwner{handles: make(map[int]windows.Handle), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *hookTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + if _, ok := owner.handles[pid]; ok { + return nil + } + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return err + } + owner.handles[pid] = handle + return nil +} + +func (owner *hookTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + handle, ok := owner.handles[pid] + if !ok { + return errors.New("test process identity was not retained") + } + wait, err := windows.WaitForSingleObject(handle, uint32(timeout/time.Millisecond)) + if err != nil { + return err + } + if wait != windows.WAIT_OBJECT_0 { + return fmt.Errorf("process is still running (wait result %#x)", wait) + } + return nil +} + +func (owner *hookTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid, handle := range owner.handles { + wait, err := windows.WaitForSingleObject(handle, 2_000) + if err == nil && wait == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(handle, 1); err != nil { + t.Errorf("terminate test process %d: %v", pid, err) + } + } + } + for pid, handle := range owner.handles { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + if err := windows.CloseHandle(handle); err != nil { + t.Errorf("close test process %d handle: %v", pid, err) + } + delete(owner.handles, pid) + } +} + +func (owner *hookTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil { + _ = owner.retain(pid) + } + } +} diff --git a/internal/perfbench/taskbench.go b/internal/perfbench/taskbench.go index a922bd998..73c3335bb 100644 --- a/internal/perfbench/taskbench.go +++ b/internal/perfbench/taskbench.go @@ -338,8 +338,11 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { cmd.Stdout = &stdout cmd.Stderr = &stderr runErr := execution.RunCommand(ctx, cmd) - if errors.Is(runErr, exec.ErrWaitDelay) { - return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + if !runEndCanReconcile(runErr) { + if errors.Is(runErr, exec.ErrWaitDelay) { + return TaskOutcome{Err: fmt.Errorf("zero exec output cleanup failed: %w", runErr)} + } + return TaskOutcome{Err: fmt.Errorf("zero exec command failed: %w", runErr)} } // The terminal run_end exit code is authoritative for pass/fail: a non-zero @@ -370,6 +373,17 @@ func NewExecRunner(binary string, extraArgs ...string) TaskRunner { } } +// runEndCanReconcile reports whether a command result contains only an ordinary +// process exit status. A run_end may explain success or an *exec.ExitError, but +// it must not hide cancellation, startup, process-tree, or output-cleanup errors. +func runEndCanReconcile(err error) bool { + if err == nil { + return true + } + _, ok := execution.AsPureExitError(err) + return ok +} + func buildExecArgs(task BenchTask, rc RunContext, extraArgs []string) []string { args := []string{"exec", "--output-format", "stream-json"} if model := strings.TrimSpace(rc.Model); model != "" { diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index e42d0af82..f01667798 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -3,12 +3,15 @@ package perfbench import ( "context" "errors" + "fmt" "os" + "os/exec" "path/filepath" "runtime" "strconv" "strings" "testing" + "time" ) func sampleTaskSet() TaskSet { @@ -276,6 +279,40 @@ func writeExecStub(t *testing.T, body string) string { return path } +func writeBlockingExecStub(t *testing.T) string { + t.Helper() + dir := t.TempDir() + source := filepath.Join(dir, "main.go") + if err := os.WriteFile(source, []byte(`package main + +import ( + "fmt" + "os" + "time" +) + +func main() { + fmt.Println("{\"type\":\"run_end\",\"exitCode\":0}") + if ready := os.Getenv("PERFBENCH_BLOCKING_STUB_READY"); ready != "" { + if err := os.WriteFile(ready, nil, 0600); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) +} +`), 0o600); err != nil { + t.Fatalf("write blocking exec stub: %v", err) + } + binary := filepath.Join(dir, "zero-stub") + if runtime.GOOS == "windows" { + binary += ".exe" + } + if output, err := exec.Command("go", "build", "-o", binary, source).CombinedOutput(); err != nil { + t.Fatalf("build blocking exec stub: %v\n%s", err, output) + } + return binary +} + func TestNewExecRunnerNonZeroRunEndIsFailNotError(t *testing.T) { // A non-zero run_end exit code is a normal task failure, not a harness error, // even though the process itself exits non-zero. @@ -292,6 +329,78 @@ exit 1 } } +func TestRunEndCanReconcile(t *testing.T) { + exitErr := &exec.ExitError{} + tests := []struct { + name string + err error + want bool + }{ + {name: "success", want: true}, + {name: "exit error", err: exitErr, want: true}, + {name: "joined exit errors", err: errors.Join(exitErr, &exec.ExitError{}), want: true}, + {name: "ordinary error", err: errors.New("startup failed")}, + {name: "canceled", err: context.Canceled}, + {name: "deadline", err: context.DeadlineExceeded}, + {name: "wait delay", err: exec.ErrWaitDelay}, + {name: "exit plus cancellation", err: errors.Join(exitErr, context.Canceled)}, + {name: "wrapped exit error", err: fmt.Errorf("attachment failed: %w", exitErr)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := runEndCanReconcile(test.err); got != test.want { + t.Fatalf("runEndCanReconcile(%v) = %v, want %v", test.err, got, test.want) + } + }) + } +} + +func TestNewExecRunnerRunEndCannotHideContextFailure(t *testing.T) { + stub := writeBlockingExecStub(t) + tests := []struct { + name string + context func(t *testing.T) context.Context + wantErr error + }{ + { + name: "cancellation", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + timer := time.AfterFunc(time.Second, cancel) + t.Cleanup(func() { timer.Stop() }) + return ctx + }, + wantErr: context.Canceled, + }, + { + name: "deadline", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + return ctx + }, + wantErr: context.DeadlineExceeded, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ready := filepath.Join(t.TempDir(), "ready") + t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) + outcome := NewExecRunner(stub)(test.context(t), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + if _, err := os.Stat(ready); err != nil { + t.Fatalf("stub did not emit run_end before the context failed: %v", err) + } + if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { + t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) + } + if outcome.Passed { + t.Fatal("context failure must not reach task pass accounting") + } + }) + } +} + func TestNewExecRunnerMissingRunEndFailsClosed(t *testing.T) { // A clean exit with no terminal run_end event is a harness error: we cannot // claim the task passed when the agent never reported a terminal event. diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 1e30db2cf..ab4b5f87d 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -669,12 +669,18 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner { runErr := execution.RunCommand(ctx, cmd) wallMs := float64(time.Since(start).Microseconds()) / 1000 - exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) outcome := TurnTaskOutcome{WallMs: wallMs} - if errors.Is(runErr, exec.ErrWaitDelay) { - outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + if !runEndCanReconcile(runErr) { + if errors.Is(runErr, exec.ErrWaitDelay) { + outcome.Err = fmt.Errorf("zero exec output cleanup failed: %w", runErr) + } else { + outcome.Err = fmt.Errorf("zero exec command failed: %w", runErr) + } return outcome - } else if haveExit && exitCode != 0 { + } + + exitCode, haveExit := streamJSONExitCode(outBuf.Bytes()) + if haveExit && exitCode != 0 { outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode) } else if !haveExit { detail := strings.TrimSpace(errBuf.String()) diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 2be6d1a1f..ab9f372d4 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -657,6 +657,53 @@ exit 0 } } +func TestNewTurnExecRunnerRunEndCannotHideContextFailure(t *testing.T) { + task := BenchTask{ID: "context-failure", Prompt: "p", WorkspaceFixture: t.TempDir()} + stub := writeBlockingExecStub(t) + tests := []struct { + name string + context func(t *testing.T) context.Context + wantErr error + }{ + { + name: "cancellation", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + timer := time.AfterFunc(time.Second, cancel) + t.Cleanup(func() { timer.Stop() }) + return ctx + }, + wantErr: context.Canceled, + }, + { + name: "deadline", + context: func(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + return ctx + }, + wantErr: context.DeadlineExceeded, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ready := filepath.Join(t.TempDir(), "ready") + t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) + outcome := NewTurnExecRunner(stub)(test.context(t), task, RunContext{Model: "m"}) + if _, err := os.Stat(ready); err != nil { + t.Fatalf("stub did not emit run_end before the context failed: %v", err) + } + if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { + t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) + } + if outcome.Passed || outcome.VerifyErr != "" { + t.Fatalf("context failure must precede oracle accounting, got %#v", outcome) + } + }) + } +} + // assertVerifyFailed asserts an outcome failed specifically because the oracle // rejected the work — Passed is false, there is no harness error (Err nil), and // VerifyErr carries the surfaced failure detail. This is stronger than merely @@ -810,6 +857,7 @@ func TestOracleAuthoritativeOnIncompleteExit(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, `sed 's/const MaxRetries = 3/const RetryLimit = 3/' main.go > .zero-tmp && mv .zero-tmp main.go echo '{"type":"run_end","exitCode":4}' +exit 4 `) if outcome.Err != nil { t.Fatalf("incomplete-exit with a correct edit should pass, got harness error: %v", outcome.Err) @@ -836,7 +884,8 @@ func TestNonIncompleteExitStaysAuthoritative(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, fmt.Sprintf(`sed 's/const MaxRetries = 3/const RetryLimit = 3/' main.go > .zero-tmp && mv .zero-tmp main.go echo '{"type":"run_end","exitCode":%d}' -`, code)) +exit %d +`, code, code)) if outcome.Err != nil { t.Fatalf("a nonzero exit should be a task fail, not a harness error: %v", outcome.Err) } @@ -860,6 +909,7 @@ echo '{"type":"run_end","exitCode":%d}' func TestIncompleteExitStillFailsWhenOracleFails(t *testing.T) { task := loadBaselineTask(t, "edit-01") outcome := runTurnStub(t, task, `echo '{"type":"run_end","exitCode":4}' +exit 4 `) assertVerifyFailed(t, "incomplete exit with no edit applied", outcome) } @@ -872,6 +922,7 @@ func TestIncompleteExitStillFailsWhenOracleFails(t *testing.T) { func TestNonzeroExitStillFailsLatencyOnly(t *testing.T) { task := loadBaselineTask(t, "longproc-01") outcome := runTurnStub(t, task, `echo '{"type":"run_end","exitCode":4}' +exit 4 `) if outcome.Err != nil { t.Fatalf("latency-only nonzero exit should be a verify fail, not a harness error: %v", outcome.Err) diff --git a/internal/verify/process_unix_test.go b/internal/verify/process_unix_test.go new file mode 100644 index 000000000..cdd5ff7f4 --- /dev/null +++ b/internal/verify/process_unix_test.go @@ -0,0 +1,81 @@ +//go:build !windows + +package verify + +import ( + "errors" + "os" + "strconv" + "syscall" + "testing" + "time" +) + +type verifyTestProcessOwner struct { + pids map[int]struct{} + exited map[int]struct{} + stopFile string + pidFiles []string +} + +func newVerifyTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *verifyTestProcessOwner { + t.Helper() + owner := &verifyTestProcessOwner{pids: make(map[int]struct{}), exited: make(map[int]struct{}), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *verifyTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + owner.pids[pid] = struct{}{} + return nil +} + +func (owner *verifyTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := syscall.Kill(pid, syscall.Signal(0)) + if errors.Is(err, syscall.ESRCH) { + delete(owner.pids, pid) + owner.exited[pid] = struct{}{} + return nil + } + if err != nil { + return err + } + if time.Now().After(deadline) { + return errors.New("process is still running") + } + time.Sleep(10 * time.Millisecond) + } +} + +func (owner *verifyTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid := range owner.pids { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + } +} + +func (owner *verifyTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil && pid > 0 && pid != os.Getpid() { + if _, exited := owner.exited[pid]; !exited { + owner.pids[pid] = struct{}{} + } + } + } +} diff --git a/internal/verify/process_windows_test.go b/internal/verify/process_windows_test.go new file mode 100644 index 000000000..a8612a25e --- /dev/null +++ b/internal/verify/process_windows_test.go @@ -0,0 +1,95 @@ +//go:build windows + +package verify + +import ( + "errors" + "fmt" + "os" + "strconv" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +type verifyTestProcessOwner struct { + handles map[int]windows.Handle + stopFile string + pidFiles []string +} + +func newVerifyTestProcessOwner(t *testing.T, stopFile string, pidFiles ...string) *verifyTestProcessOwner { + t.Helper() + owner := &verifyTestProcessOwner{handles: make(map[int]windows.Handle), stopFile: stopFile, pidFiles: pidFiles} + t.Cleanup(func() { owner.cleanup(t) }) + return owner +} + +func (owner *verifyTestProcessOwner) retain(pid int) error { + if pid <= 0 || pid == os.Getpid() { + return errors.New("invalid test process PID") + } + if _, ok := owner.handles[pid]; ok { + return nil + } + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return err + } + owner.handles[pid] = handle + return nil +} + +func (owner *verifyTestProcessOwner) awaitExit(pid int, timeout time.Duration) error { + handle, ok := owner.handles[pid] + if !ok { + return errors.New("test process identity was not retained") + } + wait, err := windows.WaitForSingleObject(handle, uint32(timeout/time.Millisecond)) + if err != nil { + return err + } + if wait != windows.WAIT_OBJECT_0 { + return fmt.Errorf("process is still running (wait result %#x)", wait) + } + return nil +} + +func (owner *verifyTestProcessOwner) cleanup(t *testing.T) { + t.Helper() + if err := os.WriteFile(owner.stopFile, nil, 0o600); err != nil { + t.Errorf("request test process stop: %v", err) + } + owner.retainPIDFiles() + for pid, handle := range owner.handles { + wait, err := windows.WaitForSingleObject(handle, 2_000) + if err == nil && wait == uint32(windows.WAIT_TIMEOUT) { + if err := windows.TerminateProcess(handle, 1); err != nil { + t.Errorf("terminate test process %d: %v", pid, err) + } + } + } + for pid, handle := range owner.handles { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Errorf("wait for test process %d: %v", pid, err) + } + if err := windows.CloseHandle(handle); err != nil { + t.Errorf("close test process %d handle: %v", pid, err) + } + delete(owner.handles, pid) + } +} + +func (owner *verifyTestProcessOwner) retainPIDFiles() { + for _, path := range owner.pidFiles { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := strconv.Atoi(string(data)) + if err == nil { + _ = owner.retain(pid) + } + } +} diff --git a/internal/verify/verify.go b/internal/verify/verify.go index a571cada8..3464d0f55 100644 --- a/internal/verify/verify.go +++ b/internal/verify/verify.go @@ -3,7 +3,6 @@ package verify import ( "bytes" "context" - "errors" "fmt" "os" "os/exec" @@ -310,8 +309,7 @@ func defaultRunner(ctx context.Context, dir string, command []string, timeout ti exitCode := 0 if err != nil { exitCode = -1 - var exitError *exec.ExitError - if errors.As(err, &exitError) { + if exitError, ok := execution.AsPureExitError(err); ok { exitCode = exitError.ExitCode() err = nil } diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go index b8792ca09..2e49e1634 100644 --- a/internal/verify/verify_test.go +++ b/internal/verify/verify_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "time" @@ -16,36 +17,115 @@ import ( func TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { switch os.Getenv("ZERO_VERIFY_TREE_HELPER") { case "parent": - if err := os.Setenv("ZERO_VERIFY_TREE_HELPER", "grandchild"); err != nil { + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_PARENT_PID_FILE"), []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { os.Exit(2) } child := exec.Command(os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$") - child.Env = os.Environ() + child.Env = append(os.Environ(), + "ZERO_VERIFY_TREE_HELPER=grandchild", + "ZERO_VERIFY_TREE_STOP_FILE="+os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), + ) child.Stdout = os.Stdout child.Stderr = os.Stderr if err := child.Start(); err != nil { os.Exit(3) } - select {} + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_GRANDCHILD_PID_FILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(4) + } + if err := os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_READY_FILE"), []byte("ready"), 0o600); err != nil { + _ = os.WriteFile(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), nil, 0o600) + _ = child.Wait() + os.Exit(5) + } + waitForVerifyTreeStop(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), 30*time.Second) + _ = child.Wait() + return case "grandchild": - time.Sleep(30 * time.Second) + waitForVerifyTreeStop(os.Getenv("ZERO_VERIFY_TREE_STOP_FILE"), 30*time.Second) return } + root := t.TempDir() + parentPIDFile := filepath.Join(root, "parent.pid") + grandchildPIDFile := filepath.Join(root, "grandchild.pid") + readyFile := filepath.Join(root, "ready") + stopFile := filepath.Join(root, "stop") + owner := newVerifyTestProcessOwner(t, stopFile, parentPIDFile, grandchildPIDFile) t.Setenv("ZERO_VERIFY_TREE_HELPER", "parent") - plan := Plan{Root: t.TempDir(), Checks: []Check{{ + t.Setenv("ZERO_VERIFY_TREE_PARENT_PID_FILE", parentPIDFile) + t.Setenv("ZERO_VERIFY_TREE_GRANDCHILD_PID_FILE", grandchildPIDFile) + t.Setenv("ZERO_VERIFY_TREE_READY_FILE", readyFile) + t.Setenv("ZERO_VERIFY_TREE_STOP_FILE", stopFile) + plan := Plan{Root: root, Checks: []Check{{ ID: "tree.timeout", Name: "process tree timeout", Command: []string{os.Args[0], "-test.run=^TestRunDefaultRunnerTimeoutKillsGrandchildHoldingOutput$"}, }}} + reportChannel := make(chan Report, 1) + go func() { + reportChannel <- Run(context.Background(), plan, RunOptions{TimeoutMS: 3000}) + }() + parentPID, grandchildPID := awaitVerifyTreeReady(t, owner, readyFile, parentPIDFile, grandchildPIDFile) started := time.Now() - report := Run(context.Background(), plan, RunOptions{TimeoutMS: 100}) - if elapsed := time.Since(started); elapsed > 2*time.Second { + var report Report + select { + case report = <-reportChannel: + case <-time.After(6 * time.Second): + t.Fatal("defaultRunner did not return within six seconds after its timeout") + } + if elapsed := time.Since(started); elapsed > 4*time.Second { t.Fatalf("defaultRunner remained blocked by grandchild output handles for %s", elapsed) } if report.OK || len(report.Results) != 1 || report.Results[0].Status == StatusPass { t.Fatalf("timed-out defaultRunner command unexpectedly passed: %#v", report) } + for role, pid := range map[string]int{"parent": parentPID, "grandchild": grandchildPID} { + if err := owner.awaitExit(pid, 2*time.Second); err != nil { + t.Fatalf("%s process %d survived verify cancellation: %v", role, pid, err) + } + } +} + +func waitForVerifyTreeStop(stopFile string, lifetime time.Duration) { + deadline := time.Now().Add(lifetime) + for time.Now().Before(deadline) { + if _, err := os.Stat(stopFile); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } +} + +func awaitVerifyTreeReady(t *testing.T, owner *verifyTestProcessOwner, readyFile string, pidFiles ...string) (int, int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(readyFile); err == nil { + pids := make([]int, 0, len(pidFiles)) + for _, path := range pidFiles { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read helper PID handoff %q: %v", path, err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatalf("parse helper PID handoff %q: %v", data, err) + } + if err := owner.retain(pid); err != nil { + t.Fatalf("retain helper process %d: %v", pid, err) + } + pids = append(pids, pid) + } + return pids[0], pids[1] + } + if time.Now().After(deadline) { + t.Fatal("process-tree helper did not hand off parent and grandchild identities") + } + time.Sleep(10 * time.Millisecond) + } } func TestDetectPlanFindsBunAndGoChecks(t *testing.T) { From d6e8f02c11163db6f1cb88b2a4e398dd7ef43893 Mon Sep 17 00:00:00 2001 From: Amp Date: Mon, 7 Sep 2026 18:12:27 +0000 Subject: [PATCH 15/17] fix(execution): bound configured hook process lifecycles Amp-Thread-ID: https://ampcode.com/threads/T-01a07cfc-2705-74bc-a369-6e0a867f0be9 Co-authored-by: Pierre Bruno --- internal/execution/command_context.go | 9 +- internal/execution/command_context_test.go | 10 ++ internal/execution/runner.go | 2 +- internal/hooks/dispatch_test.go | 122 ++++++++++++++++++++- internal/perfbench/taskbench_test.go | 99 +++++++++++++---- internal/perfbench/turn_bench_test.go | 26 +---- 6 files changed, 220 insertions(+), 48 deletions(-) diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 015a7e151..84bb0dc67 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -16,6 +16,9 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if ctx == nil { ctx = context.Background() } + if err := ctx.Err(); err != nil { + return err + } tree, err := prepareCommandTree(command) if err != nil { return err @@ -23,7 +26,11 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { defer func() { err = errors.Join(err, tree.close()) }() command.WaitDelay = processWaitDelay - command.Cancel = tree.cancel + // Adapters may return exec.Command, which rejects a non-nil Cancel. + // The watcher below also owns cancellation for commands without that hook. + if command.Cancel != nil { + command.Cancel = tree.cancel + } if err := command.Start(); err != nil { _ = tree.attach(nil) return err diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index 73cf8fd69..a2e6e6686 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -11,6 +11,16 @@ import ( "time" ) +func TestRunCommandCanceledBeforePlainCommandStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + command := exec.Command(os.Args[0], "-test.run=^$") + err := RunCommand(ctx, command) + if !errors.Is(err, context.Canceled) || command.Process != nil { + t.Fatalf("canceled command must not start: err=%v process=%v", err, command.Process) + } +} + func TestRunCommandKillsDescendantAfterRootExit(t *testing.T) { switch os.Getenv("ZERO_COMMAND_TREE_HELPER") { case "root": diff --git a/internal/execution/runner.go b/internal/execution/runner.go index 7da175419..94af93c1c 100644 --- a/internal/execution/runner.go +++ b/internal/execution/runner.go @@ -89,7 +89,7 @@ func (runner *Runner) ExecuteCaptured(ctx context.Context, input CapturedRequest stderr := &capturedBuffer{limit: maxCapturedStreamBytes} prepared.Command.Stdout = stdout prepared.Command.Stderr = stderr - runErr := prepared.Command.Run() + runErr := RunCommand(ctx, prepared.Command) report, reportErr := AdapterReport{}, error(nil) if prepared.Report != nil { report, reportErr = prepared.Report() diff --git a/internal/hooks/dispatch_test.go b/internal/hooks/dispatch_test.go index 7e4de503b..c058a6d81 100644 --- a/internal/hooks/dispatch_test.go +++ b/internal/hooks/dispatch_test.go @@ -2,6 +2,8 @@ package hooks import ( "context" + "fmt" + "io" "os" "os/exec" "path/filepath" @@ -16,11 +18,16 @@ import ( type hookExecutionPreparer struct { request execution.Request + report func() (execution.AdapterReport, error) + cleanup func() } -func (preparer *hookExecutionPreparer) PrepareExecution(_ context.Context, request execution.Request) (execution.PreparedCommand, error) { +func (preparer *hookExecutionPreparer) PrepareExecution(ctx context.Context, request execution.Request) (execution.PreparedCommand, error) { preparer.request = request - return execution.PreparedCommand{Command: exec.Command(request.Command.Name, request.Command.Args...)}, nil + command := exec.CommandContext(ctx, request.Command.Name, request.Command.Args...) + command.Env = request.Command.Env + command.Dir = request.WorkingDirectory + return execution.PreparedCommand{Command: command, Report: preparer.report, Cleanup: preparer.cleanup}, nil } func beforeToolConfig(hooks ...Definition) Config { @@ -53,6 +60,16 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { _ = child.Wait() os.Exit(5) } + if exitFile := os.Getenv("ZERO_HOOK_TREE_EXIT_FILE"); exitFile != "" { + input, err := io.ReadAll(os.Stdin) + if err != nil { + os.Exit(6) + } + fmt.Fprintln(os.Stdout, string(input)) + fmt.Fprintln(os.Stderr, "hook diagnostic") + waitForHookTreeStop(exitFile, 30*time.Second) + os.Exit(0) + } waitForHookTreeStop(os.Getenv("ZERO_HOOK_TREE_STOP_FILE"), 30*time.Second) _ = child.Wait() return @@ -108,6 +125,105 @@ func TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { } } +// The test injects deadline expiry only after the process handoffs, independently +// of startup speed and of the watchdog that bounds a broken execution runner. +type hookDeadlineContext struct{ context.Context } + +// Prevent context.WithTimeout from bypassing Err via the embedded cancelCtx. +func (hookDeadlineContext) Value(any) any { return nil } + +func (ctx hookDeadlineContext) Err() error { + if ctx.Context.Err() != nil { + return context.DeadlineExceeded + } + return nil +} + +func TestDispatchConfiguredRunnerTimeoutKillsGrandchildHoldingOutput(t *testing.T) { + root := t.TempDir() + parentFile, childFile := filepath.Join(root, "parent.pid"), filepath.Join(root, "child.pid") + ready, stop, exit := filepath.Join(root, "ready"), filepath.Join(root, "stop"), filepath.Join(root, "exit") + owner := newHookTestProcessOwner(t, stop, parentFile, childFile) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + // Registered before launch: release the root and pipe holder even when the + // production lifecycle regresses or a readiness assertion aborts the test. + t.Cleanup(func() { + cancel() + for _, path := range []string{exit, stop} { + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Error(err) + } + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("dispatch did not finish after independent fixture cleanup") + } + }) + audit, err := NewAuditStore(AuditStoreOptions{AuditPath: filepath.Join(root, "audit.jsonl")}) + if err != nil { + t.Fatal(err) + } + reported, cleaned := false, false + preparer := &hookExecutionPreparer{ + report: func() (execution.AdapterReport, error) { + reported = true + return execution.AdapterReport{}, nil + }, + cleanup: func() { cleaned = true }, + } + dispatcher := NewDispatcher(DispatcherOptions{ + Config: beforeToolConfig(Definition{ID: "tree", Event: EventBeforeTool, Enabled: true, + Command: os.Args[0], Args: []string{"-test.run=^TestExecCommandRunnerTimeoutKillsGrandchildHoldingOutput$"}}), + Execution: execution.NewRunner(preparer), Audit: audit, Cwd: root, + Timeout: time.Minute, + Env: append(os.Environ(), "ZERO_HOOK_TREE_HELPER=parent", + "ZERO_HOOK_TREE_PARENT_PID_FILE="+parentFile, "ZERO_HOOK_TREE_GRANDCHILD_PID_FILE="+childFile, + "ZERO_HOOK_TREE_READY_FILE="+ready, "ZERO_HOOK_TREE_STOP_FILE="+stop, "ZERO_HOOK_TREE_EXIT_FILE="+exit), + }) + results := make(chan DispatchOutcome, 1) + go func() { + defer close(done) + results <- dispatcher.Dispatch(hookDeadlineContext{ctx}, DispatchInput{Event: EventBeforeTool, Payload: "payload"}) + }() + parentPID, childPID := awaitHookTreeReady(t, owner, ready, parentFile, childFile) + if err := os.WriteFile(exit, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := owner.awaitExit(parentPID, 2*time.Second); err != nil { + t.Fatalf("root did not exit before deadline: %v", err) + } + cancel() + var outcome DispatchOutcome + select { + case outcome = <-results: + case <-time.After(4 * time.Second): + t.Fatal("configured hook remained blocked by inherited output after deadline") + } + if !outcome.Blocked || outcome.Ran != 1 || !strings.Contains(outcome.Reason, "hook timed out") { + t.Fatalf("deadline not reported: %#v", outcome) + } + if len(outcome.Messages) != 1 || outcome.Messages[0] != `"payload"` { + t.Fatalf("stdin/output not preserved: %#v", outcome) + } + if !reported || !cleaned { + t.Fatalf("adapter callbacks not preserved: report=%v cleanup=%v", reported, cleaned) + } + if err := owner.awaitExit(childPID, 2*time.Second); err != nil { + t.Fatalf("grandchild survived configured hook deadline: %v", err) + } + events, err := audit.ReadEvents() + if err != nil { + t.Fatal(err) + } + if len(events) != 2 || events[0].Type != "hook_execution_started" || events[1].Status != AuditBlocked || + len(events[1].Results) != 1 || strings.TrimSpace(events[1].Results[0].Stdout) != `"payload"` || + strings.TrimSpace(events[1].Results[0].Stderr) != "hook diagnostic" { + t.Fatalf("audit/output not preserved: %#v", events) + } +} + func waitForHookTreeStop(stopFile string, lifetime time.Duration) { deadline := time.Now().Add(lifetime) for time.Now().Before(deadline) { @@ -120,7 +236,7 @@ func waitForHookTreeStop(stopFile string, lifetime time.Duration) { func awaitHookTreeReady(t *testing.T, owner *hookTestProcessOwner, readyFile string, pidFiles ...string) (int, int) { t.Helper() - deadline := time.Now().Add(2 * time.Second) + deadline := time.Now().Add(15 * time.Second) for { if _, err := os.Stat(readyFile); err == nil { pids := make([]int, 0, len(pidFiles)) diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index f01667798..bc340718a 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -298,7 +298,12 @@ func main() { panic(err) } } - time.Sleep(3 * time.Second) + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); { + if _, err := os.Stat(os.Getenv("PERFBENCH_BLOCKING_STUB_STOP")); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } } `), 0o600); err != nil { t.Fatalf("write blocking exec stub: %v", err) @@ -313,6 +318,72 @@ func main() { return binary } +// Override Err only after Done closes, allowing both context failures to be +// injected at the readiness handoff rather than racing process startup. +type stubFailureContext struct { + context.Context + failure error +} + +func (ctx stubFailureContext) Err() error { + if ctx.Context.Err() != nil { + return ctx.failure + } + return nil +} + +func runAfterStubReady[T any](t *testing.T, failure error, run func(context.Context) T) T { + t.Helper() + root := t.TempDir() + ready, stop := filepath.Join(root, "ready"), filepath.Join(root, "stop") + t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) + t.Setenv("PERFBENCH_BLOCKING_STUB_STOP", stop) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan T, 1) + done := make(chan struct{}) + t.Cleanup(func() { + cancel() + // Independent of the production process-tree cleanup being tested. + if err := os.WriteFile(stop, nil, 0o600); err != nil { + t.Error(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("stub did not stop after independent cleanup") + } + }) + go func() { + defer close(done) + result <- run(stubFailureContext{Context: ctx, failure: failure}) + }() + watchdog := time.NewTimer(15 * time.Second) + defer watchdog.Stop() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + if _, err := os.Stat(ready); err == nil { + break + } + select { + case <-result: + t.Fatal("runner returned before run_end readiness handoff") + case <-watchdog.C: + t.Fatal("stub did not emit run_end before watchdog") + case <-ticker.C: + } + } + cancel() + select { + case outcome := <-result: + return outcome + case <-time.After(4 * time.Second): + t.Fatal("runner did not return after context failure") + } + var zero T + return zero +} + func TestNewExecRunnerNonZeroRunEndIsFailNotError(t *testing.T) { // A non-zero run_end exit code is a normal task failure, not a harness error, // even though the process itself exits non-zero. @@ -359,38 +430,22 @@ func TestNewExecRunnerRunEndCannotHideContextFailure(t *testing.T) { stub := writeBlockingExecStub(t) tests := []struct { name string - context func(t *testing.T) context.Context wantErr error }{ { - name: "cancellation", - context: func(t *testing.T) context.Context { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - timer := time.AfterFunc(time.Second, cancel) - t.Cleanup(func() { timer.Stop() }) - return ctx - }, + name: "cancellation", wantErr: context.Canceled, }, { - name: "deadline", - context: func(t *testing.T) context.Context { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - t.Cleanup(cancel) - return ctx - }, + name: "deadline", wantErr: context.DeadlineExceeded, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ready := filepath.Join(t.TempDir(), "ready") - t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) - outcome := NewExecRunner(stub)(test.context(t), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) - if _, err := os.Stat(ready); err != nil { - t.Fatalf("stub did not emit run_end before the context failed: %v", err) - } + outcome := runAfterStubReady(t, test.wantErr, func(ctx context.Context) TaskOutcome { + return NewExecRunner(stub)(ctx, BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + }) if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) } diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index ab9f372d4..c261b5f93 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -662,38 +662,22 @@ func TestNewTurnExecRunnerRunEndCannotHideContextFailure(t *testing.T) { stub := writeBlockingExecStub(t) tests := []struct { name string - context func(t *testing.T) context.Context wantErr error }{ { - name: "cancellation", - context: func(t *testing.T) context.Context { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - timer := time.AfterFunc(time.Second, cancel) - t.Cleanup(func() { timer.Stop() }) - return ctx - }, + name: "cancellation", wantErr: context.Canceled, }, { - name: "deadline", - context: func(t *testing.T) context.Context { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - t.Cleanup(cancel) - return ctx - }, + name: "deadline", wantErr: context.DeadlineExceeded, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ready := filepath.Join(t.TempDir(), "ready") - t.Setenv("PERFBENCH_BLOCKING_STUB_READY", ready) - outcome := NewTurnExecRunner(stub)(test.context(t), task, RunContext{Model: "m"}) - if _, err := os.Stat(ready); err != nil { - t.Fatalf("stub did not emit run_end before the context failed: %v", err) - } + outcome := runAfterStubReady(t, test.wantErr, func(ctx context.Context) TurnTaskOutcome { + return NewTurnExecRunner(stub)(ctx, task, RunContext{Model: "m"}) + }) if outcome.Err == nil || !errors.Is(outcome.Err, test.wantErr) { t.Fatalf("run_end must not hide %v, got %#v", test.wantErr, outcome) } From aa20bc19d4211045c08fdb62aca6b3191f0e7c03 Mon Sep 17 00:00:00 2001 From: Amp Date: Sat, 12 Sep 2026 23:13:41 +0000 Subject: [PATCH 16/17] test(plugins): allow race runtime exit delay in origin fixture The captured runner now enforces deadlines even for plain exec.Command adapters. The origin-wiring fixture's one-second timeout collided with the race runtime's one-second exit delay. Allow a 15-second watchdog without changing origin, mode, or successful-exit assertions. The old test failed 3/3 race runs on the PR head and passed 3/3 on upstream; the fix passes 10/10 with default race options. Validation: full tests and make test, seven-package focused race suite, fmt-check, vet, release build/smoke, vulncheck, and diff hygiene pass. Advisory lint retains four pre-existing upstream staticcheck suggestions. Amp-Thread-ID: https://ampcode.com/threads/T-01a097c8-157d-755f-80b4-f79cf525ec50 Co-authored-by: Pierre Bruno --- internal/plugins/activate_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/plugins/activate_test.go b/internal/plugins/activate_test.go index 7d100c9e7..385570e9a 100644 --- a/internal/plugins/activate_test.go +++ b/internal/plugins/activate_test.go @@ -572,7 +572,10 @@ func TestPluginCommandUsesTypedPluginExecutionOrigin(t *testing.T) { } preparer := &pluginExecutionPreparer{} command := pluginCommand{Command: os.Args[0], Args: []string{"-test.run=^$"}, Cwd: t.TempDir()} - output := execPluginCommandWithExecution(context.Background(), execution.NewRunner(preparer), command, time.Second) + // This checks origin wiring, not deadline enforcement. Allow startup and + // the race runtime's one-second exit delay now that even a plain exec.Command + // returned by the preparer is subject to the captured execution deadline. + output := execPluginCommandWithExecution(context.Background(), execution.NewRunner(preparer), command, 15*time.Second) if output.Err != nil || output.ExitCode != 0 { t.Fatalf("plugin execution output = %#v", output) } From 173ddc35c77c3ee02ed258203f53fdac2c2f7227 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sun, 13 Sep 2026 13:29:31 +0200 Subject: [PATCH 17/17] fix(process): preserve benchmark output drain failures Amp-Thread-ID: https://ampcode.com/threads/T-01a09a80-5633-77ee-81f5-d56cfb50956f Co-authored-by: Amp --- internal/execution/command_context.go | 85 ++++++++++++++++++++++ internal/execution/command_context_test.go | 32 ++++++++ internal/perfbench/perfbench.go | 8 +- internal/perfbench/perfbench_test.go | 32 +++++++- internal/perfbench/taskbench_test.go | 15 ++++ internal/perfbench/turn_bench_test.go | 15 ++++ 6 files changed, 184 insertions(+), 3 deletions(-) diff --git a/internal/execution/command_context.go b/internal/execution/command_context.go index 84bb0dc67..ca3ea9b0f 100644 --- a/internal/execution/command_context.go +++ b/internal/execution/command_context.go @@ -4,7 +4,11 @@ import ( "context" "errors" "fmt" + "io" + "os" "os/exec" + "reflect" + "sync" ) // RunCommand runs a context-bound command in a retained process tree and @@ -19,6 +23,7 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if err := ctx.Err(); err != nil { return err } + drains := observeOutputDrains(command) tree, err := prepareCommandTree(command) if err != nil { return err @@ -60,8 +65,88 @@ func RunCommand(ctx context.Context, command *exec.Cmd) (err error) { if canceled.canceled { return errors.Join(waitErr, ctx.Err(), canceled.err) } + if waitErr != nil && drains.err() != nil { + // Cmd.Wait intentionally prefers an ExitError over a copying error. When + // WaitDelay forcibly closes an inherited output pipe after a nonzero root + // exit, retain that cleanup failure so result consumers cannot reconcile it + // as an ordinary command exit. + return errors.Join(waitErr, exec.ErrWaitDelay, tree.cancel()) + } if waitErr != nil { return errors.Join(waitErr, tree.cancel()) } return waitErr } + +type outputDrains struct { + stdout *drainObserver + stderr *drainObserver +} + +func observeOutputDrains(command *exec.Cmd) outputDrains { + drains := outputDrains{} + sharedOutput := sameWriter(command.Stderr, command.Stdout) + if !isFile(command.Stdout) && command.Stdout != nil { + drains.stdout = &drainObserver{writer: command.Stdout} + command.Stdout = drains.stdout + } + if sharedOutput && drains.stdout != nil { + command.Stderr = drains.stdout + drains.stderr = drains.stdout + } else if !isFile(command.Stderr) && command.Stderr != nil { + drains.stderr = &drainObserver{writer: command.Stderr} + command.Stderr = drains.stderr + } + return drains +} + +func (drains outputDrains) err() error { + if drains.stdout != nil && drains.stdout.err() != nil { + return drains.stdout.err() + } + if drains.stderr != nil { + return drains.stderr.err() + } + return nil +} + +func isFile(writer io.Writer) bool { + _, ok := writer.(*os.File) + return ok +} + +func sameWriter(left io.Writer, right io.Writer) bool { + if left == nil || right == nil { + return false + } + leftType := reflect.TypeOf(left) + return leftType == reflect.TypeOf(right) && leftType.Comparable() && left == right +} + +// drainObserver records an error from os/exec's output-copying goroutine. +// Cmd.Wait drops that error when process exit itself fails. +type drainObserver struct { + writer io.Writer + mu sync.Mutex + copyErr error +} + +func (observer *drainObserver) Write(data []byte) (int, error) { + return observer.writer.Write(data) +} + +func (observer *drainObserver) ReadFrom(reader io.Reader) (int64, error) { + written, err := io.Copy(observer.writer, reader) + if err != nil { + observer.mu.Lock() + observer.copyErr = err + observer.mu.Unlock() + } + return written, err +} + +func (observer *drainObserver) err() error { + observer.mu.Lock() + defer observer.mu.Unlock() + return observer.copyErr +} diff --git a/internal/execution/command_context_test.go b/internal/execution/command_context_test.go index a2e6e6686..c3a73a73a 100644 --- a/internal/execution/command_context_test.go +++ b/internal/execution/command_context_test.go @@ -89,6 +89,9 @@ func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { _ = child.Wait() os.Exit(3) } + if os.Getenv("ZERO_WAIT_DELAY_TREE_EXIT_CODE") == "7" { + os.Exit(7) + } return case "child": waitForCommandTreeStop(os.Getenv("ZERO_WAIT_DELAY_TREE_STOP_FILE"), 30*time.Second) @@ -120,6 +123,35 @@ func TestRunCommandKillsDescendantWhenWaitDelayExpires(t *testing.T) { child.awaitExit(t) } +func TestRunCommandPreservesWaitDelayAfterNonzeroRootExit(t *testing.T) { + root := t.TempDir() + pidFile := root + string(os.PathSeparator) + "child.pid" + stopFile := root + string(os.PathSeparator) + "stop" + child := ownHelperProcess(t, pidFile, stopFile) + cmd := exec.Command(os.Args[0], "-test.run=^TestRunCommandKillsDescendantWhenWaitDelayExpires$") + cmd.Env = append(os.Environ(), + "ZERO_WAIT_DELAY_TREE_HELPER=root", + "ZERO_WAIT_DELAY_TREE_PID_FILE="+pidFile, + "ZERO_WAIT_DELAY_TREE_STOP_FILE="+stopFile, + "ZERO_WAIT_DELAY_TREE_EXIT_CODE=7", + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + + result := runCommandAsync(context.Background(), cmd) + child.waitReady(t, 2*time.Second) + err := waitForRunCommand(t, result, 4*time.Second) + if !errors.Is(err, exec.ErrWaitDelay) { + t.Fatalf("RunCommand error = %v, want exec.ErrWaitDelay", err) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 7 { + t.Fatalf("RunCommand error = %v, want exit code 7", err) + } + child.awaitExit(t) +} + func TestRunCommandKillsDescendantAfterNonzeroRootExit(t *testing.T) { switch os.Getenv("ZERO_NONZERO_TREE_HELPER") { case "root": diff --git a/internal/perfbench/perfbench.go b/internal/perfbench/perfbench.go index 97435ff83..834bae99a 100644 --- a/internal/perfbench/perfbench.go +++ b/internal/perfbench/perfbench.go @@ -404,7 +404,7 @@ func median(sortedSamples []float64) float64 { } type timedBuffer struct { - bytes.Buffer + buffer bytes.Buffer onFirstWrite func() } @@ -412,7 +412,11 @@ func (buffer *timedBuffer) Write(data []byte) (int, error) { if len(data) > 0 { buffer.onFirstWrite() } - return buffer.Buffer.Write(data) + return buffer.buffer.Write(data) +} + +func (buffer *timedBuffer) String() string { + return buffer.buffer.String() } func commandError(command []string, err error, stdout string, stderr string) error { diff --git a/internal/perfbench/perfbench_test.go b/internal/perfbench/perfbench_test.go index 3d90e68c0..aaac8c499 100644 --- a/internal/perfbench/perfbench_test.go +++ b/internal/perfbench/perfbench_test.go @@ -6,6 +6,7 @@ import ( "os" "strings" "testing" + "time" ) func TestSummarizeSamplesSortsAndRounds(t *testing.T) { @@ -119,10 +120,39 @@ func TestPerfBenchHelperProcess(t *testing.T) { if os.Getenv("ZERO_PERF_HELPER") != "1" { return } - fmt.Println("zero 0.1.0") + if stream := os.Getenv("ZERO_PERF_HELPER_STREAM"); stream != "" { + if stream == "stderr" { + fmt.Fprintln(os.Stderr, "zero 0.1.0") + } else { + fmt.Println("zero 0.1.0") + } + time.Sleep(400 * time.Millisecond) + } else { + fmt.Println("zero 0.1.0") + } os.Exit(0) } +func TestMeasureFirstOutputCapturesFirstByteTiming(t *testing.T) { + for _, stream := range []string{"stdout", "stderr"} { + t.Run(stream, func(t *testing.T) { + command := []string{os.Args[0], "-test.run=^TestPerfBenchHelperProcess$"} + t.Setenv("ZERO_PERF_HELPER", "1") + t.Setenv("ZERO_PERF_HELPER_STREAM", stream) + sample, err := MeasureFirstOutput(context.Background(), command) + if err != nil { + t.Fatalf("MeasureFirstOutput returned error: %v", err) + } + if sample.FirstOutputMs >= 250 { + t.Fatalf("first output = %fms, want before delayed exit", sample.FirstOutputMs) + } + if sample.ProcessDrainMs < 250 { + t.Fatalf("process drain = %fms, want delayed exit time", sample.ProcessDrainMs) + } + }) + } +} + func equalFloatSlices(left []float64, right []float64) bool { if len(left) != len(right) { return false diff --git a/internal/perfbench/taskbench_test.go b/internal/perfbench/taskbench_test.go index bc340718a..d9515fb62 100644 --- a/internal/perfbench/taskbench_test.go +++ b/internal/perfbench/taskbench_test.go @@ -498,6 +498,21 @@ exit 0 } } +func TestNewExecRunnerNonzeroWaitDelayCannotReconcileRunEnd(t *testing.T) { + stub := writeExecStub(t, `sleep 3 & +echo '{"type":"run_end","exitCode":4}' +exit 4 +`) + runner := NewExecRunner(stub) + outcome := runner(context.Background(), BenchTask{ID: "t1", Prompt: "p"}, RunContext{Model: "m"}) + if outcome.Err == nil || !errors.Is(outcome.Err, exec.ErrWaitDelay) { + t.Fatalf("nonzero inherited output pipe must retain its cleanup failure, got %#v", outcome) + } + if outcome.Passed { + t.Fatal("run_end must not reconcile a nonzero command with an output cleanup failure") + } +} + func TestNewExecRunnerLaunchFailureIsHarnessError(t *testing.T) { // A binary that cannot be launched (no terminal event, process error) is a // genuine harness error. diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index c261b5f93..44617ac17 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -657,6 +658,20 @@ exit 0 } } +func TestNewTurnExecRunnerNonzeroWaitDelayCannotReconcileRunEnd(t *testing.T) { + task := BenchTask{ID: "wait-delay-nonzero", Prompt: "p", WorkspaceFixture: t.TempDir()} + outcome := runTurnStub(t, task, `sleep 3 & +echo '{"type":"run_end","exitCode":4}' +exit 4 +`) + if outcome.Err == nil || !errors.Is(outcome.Err, exec.ErrWaitDelay) { + t.Fatalf("nonzero inherited output pipe must retain its cleanup failure, got %#v", outcome) + } + if outcome.Passed || outcome.VerifyErr != "" { + t.Fatalf("output cleanup failure must precede turn accounting, got %#v", outcome) + } +} + func TestNewTurnExecRunnerRunEndCannotHideContextFailure(t *testing.T) { task := BenchTask{ID: "context-failure", Prompt: "p", WorkspaceFixture: t.TempDir()} stub := writeBlockingExecStub(t)