From 47e2fafef59f3638370db03b102acdebdbc3dc4e Mon Sep 17 00:00:00 2001 From: CMGS Date: Thu, 10 Sep 2026 01:24:39 +0900 Subject: [PATCH] test: run the cocoon and process engine scripts against shimmed tools The embedded shell scripts had no execution coverage. Each test runs a script under /bin/sh with cocoon, oras, systemctl, mount, tar and journalctl shims on PATH that log their calls and fail on demand, then asserts the call sequence, the exit code and what is left on disk: both record copies and their durable-first order, the guest address retry count, the snapshot replace on suspend, the thaw after a failed capture, the overlay mount and unmount pairing, the pull that leaves no digest behind, and the journal follow that ends with the unit or the guest. --- engine/cocoon/scripts_test.go | 768 +++++++++++++++++++++++++++++ engine/process/scripts_test.go | 872 +++++++++++++++++++++++++++++++++ 2 files changed, 1640 insertions(+) create mode 100644 engine/cocoon/scripts_test.go create mode 100644 engine/process/scripts_test.go diff --git a/engine/cocoon/scripts_test.go b/engine/cocoon/scripts_test.go new file mode 100644 index 00000000..0ac8c169 --- /dev/null +++ b/engine/cocoon/scripts_test.go @@ -0,0 +1,768 @@ +package cocoon + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cockroachdb/errors" + + "github.com/projecteru2/core/engine/workloadmeta" +) + +const ( + scriptVM = "w1" + scriptSnap = snapshotPrefix + scriptVM + scriptConsole = "/var/lib/cocoon/run/cloudhypervisor/" + scriptVM + "/console.sock" + scriptDurable = `{"id":"w1","kind":"vm","log":{"console_socket":"/old/console.sock"},"netns_pid":0}` + scriptEvents = `{"event":"UPDATED","vm":{"state":"running"}} +{"event":"DELETED","vm":{"state":"stopped"}}` + + cocoonShim = `#!/bin/sh +printf 'cocoon %s\n' "$*" >> "$STUB_LOG" +pop() { +[ -f "$1" ] || return 0 +head -n 1 "$1" +if [ "$(wc -l < "$1")" -gt 1 ]; then +tail -n +2 "$1" > "$1.tmp" +mv "$1.tmp" "$1" +fi +} +case "$1 $2" in +"vm inspect") +[ "${STUB_INSPECT:-0}" = 0 ] || exit "$STUB_INSPECT" +pop "$STUB_VM_FILE" +;; +"vm start") exit "${STUB_START:-0}";; +"vm stop") exit "${STUB_STOP:-0}";; +"vm rm") +printf '%s' "$STUB_RM_STDERR" >&2 +exit "${STUB_RM:-0}" +;; +"vm hibernate") exit "${STUB_HIBERNATE:-0}";; +"vm restore") exit "${STUB_RESTORE:-0}";; +"vm exec") +code=$(pop "$STUB_EXEC_FILE") +exit "${code:-0}" +;; +"vm status") printf '%s\n' "$STUB_EVENTS";; +"snapshot rm") exit "${STUB_SNAPSHOT:-0}";; +"image inspect") exit "${STUB_IMAGE:-1}";; +"image import") exit "${STUB_IMPORT:-0}";; +esac +exit 0 +` + + orasShim = `#!/bin/sh +printf 'oras %s\n' "$*" >> "$STUB_LOG" +[ "${STUB_ORAS_PULL:-0}" = 0 ] || exit "$STUB_ORAS_PULL" +out= +prev= +for arg in "$@"; do +if [ "$prev" = "-o" ]; then out=$arg; fi +prev=$arg +done +for name in $STUB_ORAS_FILES; do +printf 'blob\n' > "$out/$name" +done +exit 0 +` + + sedShim = `#!/bin/sh +PATH=/usr/bin:/bin +export PATH +printf 'sed %s\n' "$*" >> "$STUB_LOG" +[ "$1" = "-i" ] || exec sed "$@" +shift +n=$# +i=1 +while [ "$i" -lt "$n" ]; do +set -- "$@" "$1" +shift +i=$((i+1)) +done +file=$1 +shift +[ -f "$file" ] || exit 1 +sed "$@" "$file" > "$file.edited" || exit 1 +mv "$file.edited" "$file" +exit 0 +` + + sleepShim = `#!/bin/sh +printf 'sleep %s\n' "$*" >> "$STUB_LOG" +exit 0 +` + + journalctlShim = `#!/bin/sh +printf 'journalctl %s\n' "$*" >> "$STUB_LOG" +printf '%s\n' "$STUB_JOURNAL" +[ "${STUB_JOURNAL_HOLD:-0}" = 1 ] || exit 0 +trap 'printf "journalctl killed\n" >> "$STUB_LOG"; exit 0' TERM +/bin/sleep 5 >/dev/null 2>&1 & +wait +exit 0 +` +) + +func TestRecordScriptWritesBothCopiesOfTheRecord(t *testing.T) { + node := newScriptNode(t) + + got := node.run(t, recordScript, node.durable, node.record, storedRecord) + + if got.code != 0 { + t.Fatalf("got exit %d, want the record written: %s", got.code, got.stderr) + } + for _, path := range []string{node.durable, node.record} { + if body := node.read(t, path); body != storedRecord+"\n" { + t.Errorf("got %q in %s, want %q", body, path, storedRecord+"\n") + } + if _, err := os.Stat(path + ".tmp"); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s.tmp survived the rename: %v", path, err) + } + } +} + +func TestRecordScriptPublishesNothingWhenTheDurableCopyFails(t *testing.T) { + node := newScriptNode(t) + node.write(t, filepath.Join(node.root, "blocked"), "") + + got := node.run(t, recordScript, filepath.Join(node.root, "blocked", "w1.json"), node.record, storedRecord) + + if got.code == 0 { + t.Fatal("got exit 0, want an unwritable root reported") + } + if _, err := os.Stat(node.record); !errors.Is(err, os.ErrNotExist) { + t.Errorf("eru-agent must not see a record core could not store: %v", err) + } +} + +func TestStartScriptInspectsTheVMAroundTheBoot(t *testing.T) { + tests := []struct { + name string + keepDurable bool + start string + wantCode int + wantCalls []string + wantStdout string + }{ + { + name: "a booted vm reports its record twice", + keepDurable: true, + wantCalls: []string{ + "cocoon vm inspect " + scriptVM, + "cocoon vm start " + scriptVM, + "cocoon vm inspect " + scriptVM, + }, + wantStdout: linuxVM + "\n" + runningVM + "\n", + }, + { + name: "a vm the node lost", + wantCode: workloadmeta.NotExistsCode, + wantCalls: nil, + }, + { + name: "a vm that will not boot", + keepDurable: true, + start: "3", + wantCode: 3, + wantCalls: []string{ + "cocoon vm inspect " + scriptVM, + "cocoon vm start " + scriptVM, + }, + wantStdout: linuxVM + "\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_START"] = tt.start + node.records(t, linuxVM, runningVM) + if tt.keepDurable { + node.write(t, node.durable, scriptDurable) + } + + got := node.run(t, startScript, node.binary, scriptVM, node.durable) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + node.assertCalls(t, tt.wantCalls...) + }) + } +} + +func TestRefreshScriptRewritesTheConsoleAndThePID(t *testing.T) { + node := newScriptNode(t) + node.write(t, node.durable, scriptDurable) + + got := node.run(t, refreshScript, node.durable, node.record, scriptConsole, "4242") + + if got.code != 0 { + t.Fatalf("got exit %d, want the record refreshed: %s", got.code, got.stderr) + } + want := `{"id":"w1","kind":"vm","log":{"console_socket":"` + scriptConsole + `"},"netns_pid":4242}` + if body := node.read(t, node.durable); body != want { + t.Errorf("got %q, want %q", body, want) + } + if body := node.read(t, node.record); body != want { + t.Errorf("got %q, want the refreshed record published", body) + } +} + +func TestRefreshScriptPublishesNothingForAVMTheNodeLost(t *testing.T) { + node := newScriptNode(t) + + got := node.run(t, refreshScript, node.durable, node.record, scriptConsole, "4242") + + if got.code == 0 { + t.Fatal("got exit 0, want a missing durable record reported") + } + if _, err := os.Stat(node.record); !errors.Is(err, os.ErrNotExist) { + t.Errorf("eru-agent must not see a record of a vm that is gone: %v", err) + } +} + +func TestAddressScriptRetriesUntilTheGuestAgentAnswers(t *testing.T) { + tests := []struct { + name string + codes []string + wantCode int + wantExecs int + }{ + {"a guest that answers at once", []string{"0"}, 0, 1}, + {"a guest that boots slowly", []string{"1", "1", "0"}, 0, 3}, + {"a guest that never answers", []string{"1"}, 1, 90}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.write(t, node.env["STUB_EXEC_FILE"], strings.Join(tt.codes, "\n")+"\n") + + got := node.run(t, addressScript, node.binary, scriptVM, "10.22.0.5", "255.255.0.0", "10.22.0.1") + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + execs := 0 + want := "cocoon vm exec " + scriptVM + " -- netsh interface ip set address " + guestIface + " static 10.22.0.5 255.255.0.0 10.22.0.1" + for _, call := range node.calls(t) { + if call == want { + execs++ + } + } + if execs != tt.wantExecs { + t.Errorf("got %d attempts, want %d", execs, tt.wantExecs) + } + }) + } +} + +func TestRemoveScriptDropsTheVMAndBothRecords(t *testing.T) { + tests := []struct { + name string + keepDurable bool + force string + rm string + inspect string + wantCode int + wantCalls []string + wantGone bool + }{ + { + name: "a forced remove", + keepDurable: true, + force: "1", + wantCalls: []string{ + "cocoon vm rm --force " + scriptVM, + "cocoon snapshot rm " + scriptSnap, + }, + wantGone: true, + }, + { + name: "a graceful remove", + keepDurable: true, + force: "0", + wantCalls: []string{ + "cocoon vm rm " + scriptVM, + "cocoon snapshot rm " + scriptSnap, + }, + wantGone: true, + }, + { + name: "a vm cocoon had already dropped", + keepDurable: true, + force: "0", + rm: "1", + inspect: "1", + wantCalls: []string{ + "cocoon vm rm " + scriptVM, + "cocoon vm inspect " + scriptVM, + "cocoon snapshot rm " + scriptSnap, + }, + wantGone: true, + }, + { + name: "a vm that refused to go", + keepDurable: true, + force: "0", + rm: "1", + wantCode: 1, + wantCalls: []string{ + "cocoon vm rm " + scriptVM, + "cocoon vm inspect " + scriptVM, + }, + }, + { + name: "a vm the node lost", + force: "1", + wantCode: workloadmeta.NotExistsCode, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_RM"] = tt.rm + node.env["STUB_INSPECT"] = tt.inspect + node.env["STUB_RM_STDERR"] = "vm is running" + node.records(t, runningVM) + node.write(t, node.record, storedRecord) + if tt.keepDurable { + node.write(t, node.durable, scriptDurable) + } + + got := node.run(t, removeScript, node.binary, scriptVM, node.durable, node.record, scriptSnap, tt.force) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + node.assertCalls(t, tt.wantCalls...) + if tt.wantCode == 1 && !strings.Contains(got.stderr, "vm is running") { + t.Errorf("got %q, want the cocoon failure reported", got.stderr) + } + for _, path := range []string{node.durable, node.record} { + _, err := os.Stat(path) + if tt.wantGone && !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s survived the remove: %v", path, err) + } + if !tt.wantGone && tt.keepDurable && err != nil { + t.Errorf("a refused remove must keep %s: %v", path, err) + } + } + }) + } +} + +func TestSuspendScriptReplacesTheSnapshotItHibernatesInto(t *testing.T) { + tests := []struct { + name string + hibernate string + wantCode int + }{ + {"a guest that hibernates", "", 0}, + {"a guest that cannot hibernate", "4", 4}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_HIBERNATE"] = tt.hibernate + node.env["STUB_SNAPSHOT"] = "1" + + got := node.run(t, suspendScript, node.binary, scriptVM, scriptSnap) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + node.assertCalls(t, + "cocoon snapshot rm "+scriptSnap, + "cocoon vm hibernate --name "+scriptSnap+" "+scriptVM, + ) + }) + } +} + +func TestResumeScriptRestoresByCopyAndDropsTheSnapshot(t *testing.T) { + node := newScriptNode(t) + node.records(t, runningVM) + + got := node.run(t, resumeScript, node.binary, scriptVM, scriptSnap) + + if got.code != 0 { + t.Fatalf("got exit %d, want the guest resumed: %s", got.code, got.stderr) + } + if got.stdout != runningVM+"\n" { + t.Errorf("got %q, want the record of the resumed vm", got.stdout) + } + node.assertCalls(t, + "cocoon vm restore --restore-mode copy "+scriptVM+" "+scriptSnap, + "cocoon snapshot rm "+scriptSnap, + "cocoon vm inspect "+scriptVM, + ) +} + +func TestResumeScriptKeepsTheSnapshotWhenTheRestoreFails(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_RESTORE"] = "5" + node.records(t, runningVM) + + got := node.run(t, resumeScript, node.binary, scriptVM, scriptSnap) + + if got.code != 5 { + t.Fatalf("got exit %d, want the restore failure reported", got.code) + } + node.assertCalls(t, "cocoon vm restore --restore-mode copy "+scriptVM+" "+scriptSnap) +} + +func TestStopScriptPassesTheStopFlagsToCocoon(t *testing.T) { + tests := []struct { + name string + keepDurable bool + stop string + flags []string + wantCode int + wantCalls []string + }{ + { + name: "a graceful stop", + keepDurable: true, + flags: []string{"--timeout", "30"}, + wantCalls: []string{"cocoon vm stop --timeout 30 " + scriptVM}, + }, + { + name: "a forced stop", + keepDurable: true, + flags: []string{"--force"}, + wantCalls: []string{"cocoon vm stop --force " + scriptVM}, + }, + { + name: "a guest that will not stop", + keepDurable: true, + stop: "6", + wantCode: 6, + wantCalls: []string{"cocoon vm stop " + scriptVM}, + }, + { + name: "a vm the node lost", + wantCode: workloadmeta.NotExistsCode, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_STOP"] = tt.stop + if tt.keepDurable { + node.write(t, node.durable, scriptDurable) + } + + got := node.run(t, stopScript, slices.Concat([]string{node.binary, scriptVM, node.durable}, tt.flags)...) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + node.assertCalls(t, tt.wantCalls...) + }) + } +} + +func TestInspectScriptPrintsTheRecordAndTheVM(t *testing.T) { + tests := []struct { + name string + keepDurable bool + inspect string + wantCode int + wantStdout string + }{ + {"a vm the node runs", true, "", 0, scriptDurable + "\n" + runningVM + "\n"}, + {"a vm cocoon no longer knows", true, "7", 7, scriptDurable + "\n"}, + {"a vm the node lost", false, "", workloadmeta.NotExistsCode, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_INSPECT"] = tt.inspect + node.records(t, runningVM) + if tt.keepDurable { + node.write(t, node.durable, scriptDurable+"\n") + } + + got := node.run(t, inspectScript, node.binary, scriptVM, node.durable) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + }) + } +} + +func TestWaitScriptFollowsTheStatusStreamOfALiveVM(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_EVENTS"] = scriptEvents + node.records(t, runningVM) + + got := node.run(t, waitScript, node.binary, scriptVM) + + if got.code != 0 { + t.Fatalf("got exit %d, want the stream followed: %s", got.code, got.stderr) + } + if got.stdout != scriptEvents+"\n" { + t.Errorf("got %q, want the status events", got.stdout) + } + node.assertCalls(t, + "cocoon vm inspect "+scriptVM, + "cocoon vm status --event --format json -n 1 "+scriptVM, + ) +} + +func TestWaitScriptRefusesToWaitOnAVMCocoonDoesNotHave(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_INSPECT"] = "1" + + got := node.run(t, waitScript, node.binary, scriptVM) + + if got.code != 1 { + t.Fatalf("got exit %d, want 1: the event stream stays silent for a vm that is not there", got.code) + } + node.assertCalls(t, "cocoon vm inspect "+scriptVM) +} + +func TestImportScriptReassemblesAPartsArtifactOnce(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_FILES"] = "disk.0.part disk.1.part" + + got := node.run(t, importScript, node.binary, testImage) + + if got.code != 0 { + t.Fatalf("got exit %d, want the parts imported: %s", got.code, got.stderr) + } + calls := node.calls(t) + if len(calls) != 3 { + t.Fatalf("got calls %q, want an inspect, a pull and an import", calls) + } + tmp := strings.TrimPrefix(calls[1], "oras pull "+testImage+" -o ") + node.assertCalls(t, + "cocoon image inspect "+testImage, + "oras pull "+testImage+" -o "+tmp, + "cocoon image import "+testImage+" "+tmp+"/disk.0.part "+tmp+"/disk.1.part", + ) + if _, err := os.Stat(tmp); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the scratch dir survived the import: %v", err) + } +} + +func TestImportScriptLeavesAnImportedArtifactAlone(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_IMAGE"] = "0" + + got := node.run(t, importScript, node.binary, testImage) + + if got.code != 0 { + t.Fatalf("got exit %d, want the import skipped: %s", got.code, got.stderr) + } + node.assertCalls(t, "cocoon image inspect "+testImage) +} + +func TestImportScriptImportsNothingWhenThePullFails(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_PULL"] = "1" + + got := node.run(t, importScript, node.binary, testImage) + + if got.code == 0 { + t.Fatal("got exit 0, want a failed pull reported") + } + for _, call := range node.calls(t) { + if strings.HasPrefix(call, "cocoon image import") { + t.Errorf("got %q, want no import of an artifact that never landed", call) + } + } +} + +func TestOrasProbeAnswersForTheNodeItRunsOn(t *testing.T) { + tests := []struct { + name string + present bool + }{ + {"a node with oras", true}, + {"a node without oras", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.path = node.bin + if !tt.present { + node.remove(t, filepath.Join(node.bin, "oras")) + } + + if got := node.run(t, orasProbe); (got.code == 0) != tt.present { + t.Errorf("got exit %d, want oras present %v", got.code, tt.present) + } + }) + } +} + +func TestFollowScriptEndsTheJournalWithTheGuest(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_EVENTS"] = scriptEvents + node.env["STUB_JOURNAL"] = "console line" + node.env["STUB_JOURNAL_HOLD"] = "1" + + got := node.run(t, followScript, node.binary, scriptVM, "SYSLOG_IDENTIFIER=eru", "ERU_ID="+scriptVM, "-f", "-n", "10") + + if got.code != 0 { + t.Fatalf("got exit %d, want the follow to end cleanly: %s", got.code, got.stderr) + } + if !strings.Contains(got.stdout, "console line") { + t.Errorf("got %q, want the journal streamed", got.stdout) + } + node.assertHasCalls(t, + "journalctl SYSLOG_IDENTIFIER=eru ERU_ID="+scriptVM+" -f -n 10", + "cocoon vm status --event --format json -n 1 "+scriptVM, + "journalctl killed", + ) +} + +type scriptRun struct { + code int + stdout string + stderr string +} + +type scriptNode struct { + root string + bin string + binary string + durable string + record string + logPath string + path string + env map[string]string +} + +func newScriptNode(t *testing.T) *scriptNode { + t.Helper() + root := t.TempDir() + node := &scriptNode{ + root: root, + bin: filepath.Join(root, "bin"), + durable: filepath.Join(root, "cocoon", scriptVM+metaSuffix), + record: filepath.Join(root, "run", scriptVM+metaSuffix), + logPath: filepath.Join(root, "calls.log"), + } + node.binary = filepath.Join(node.bin, "cocoon") + node.path = node.bin + ":/usr/bin:/bin" + node.env = map[string]string{ + "STUB_LOG": node.logPath, + "STUB_VM_FILE": filepath.Join(root, "records"), + "STUB_EXEC_FILE": filepath.Join(root, "execs"), + } + for name, body := range map[string]string{ + "cocoon": cocoonShim, + "oras": orasShim, + "sed": sedShim, + "sleep": sleepShim, + "journalctl": journalctlShim, + } { + node.write(t, filepath.Join(node.bin, name), body) + if err := os.Chmod(filepath.Join(node.bin, name), 0o755); err != nil { + t.Fatalf("setup %s: %v", name, err) + } + } + return node +} + +func (n *scriptNode) run(t *testing.T, script string, args ...string) scriptRun { + t.Helper() + cmd := exec.CommandContext(t.Context(), "/bin/sh", slices.Concat([]string{"-c", script, "sh"}, args)...) + cmd.Env = slices.Concat([]string{"PATH=" + n.path, "HOME=" + n.root, "TMPDIR=" + os.TempDir()}, n.environ()) + stdout, stderr := &strings.Builder{}, &strings.Builder{} + cmd.Stdout, cmd.Stderr = stdout, stderr + err := cmd.Run() + got := scriptRun{stdout: stdout.String(), stderr: stderr.String()} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + got.code = exitErr.ExitCode() + default: + t.Fatalf("run: %v", err) + } + return got +} + +func (n *scriptNode) environ() []string { + env := make([]string, 0, len(n.env)) + for key, value := range n.env { + env = append(env, key+"="+value) + } + return env +} + +func (n *scriptNode) records(t *testing.T, records ...string) { + t.Helper() + n.write(t, n.env["STUB_VM_FILE"], strings.Join(records, "\n")+"\n") +} + +func (n *scriptNode) calls(t *testing.T) []string { + t.Helper() + body, err := os.ReadFile(n.logPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + t.Fatalf("read log: %v", err) + } + return strings.Split(strings.TrimSuffix(string(body), "\n"), "\n") +} + +func (n *scriptNode) assertCalls(t *testing.T, want ...string) { + t.Helper() + got := n.calls(t) + if !slices.Equal(got, want) { + t.Errorf("got calls %q, want %q", got, want) + } +} + +func (n *scriptNode) assertHasCalls(t *testing.T, want ...string) { + t.Helper() + got := n.calls(t) + for _, line := range want { + if !slices.Contains(got, line) { + t.Errorf("got calls %q, want one of them to be %q", got, line) + } + } +} + +func (n *scriptNode) write(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("setup %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("setup %s: %v", path, err) + } +} + +func (n *scriptNode) read(t *testing.T, path string) string { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(body) +} + +func (n *scriptNode) remove(t *testing.T, path string) { + t.Helper() + if err := os.RemoveAll(path); err != nil { + t.Fatalf("setup %s: %v", path, err) + } +} diff --git a/engine/process/scripts_test.go b/engine/process/scripts_test.go new file mode 100644 index 00000000..0ba0bb60 --- /dev/null +++ b/engine/process/scripts_test.go @@ -0,0 +1,872 @@ +package process + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cockroachdb/errors" + + "github.com/projecteru2/core/engine/workloadmeta" +) + +const ( + scriptUnit = "eru-w1.service" + scriptRef = "hub.io/ns/app:v1" + scriptDescriptor = `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:abc","size":7}` + scriptLauncher = "echo launched" + + systemctlShim = `#!/bin/sh +pop() { +[ -f "$1" ] || return 0 +head -n 1 "$1" +if [ "$(wc -l < "$1")" -gt 1 ]; then +tail -n +2 "$1" > "$1.tmp" +mv "$1.tmp" "$1" +fi +} +if [ "$1" = show ]; then +case "$*" in +*"-p LoadState"*) printf '%s\n' "$STUB_LOADSTATE";; +*"-p SubState --value"*) pop "$STUB_SUBSTATE_FILE";; +*"-p ExecMainStatus"*) printf '%s\n' "$STUB_STATUS";; +*) printf '%s' "$STUB_SHOW";; +esac +exit 0 +fi +printf 'systemctl %s\n' "$*" >> "$STUB_LOG" +for verb in $STUB_FAIL; do +if [ "$verb" = "$1" ]; then +printf 'systemctl: %s failed\n' "$1" >&2 +exit 1 +fi +done +exit 0 +` + + mountpointShim = `#!/bin/sh +printf 'mountpoint %s\n' "$*" >> "$STUB_LOG" +exit "${STUB_MOUNTPOINT:-1}" +` + + mountShim = `#!/bin/sh +printf 'mount %s\n' "$*" >> "$STUB_LOG" +exit "${STUB_MOUNT:-0}" +` + + umountShim = `#!/bin/sh +printf 'umount %s\n' "$*" >> "$STUB_LOG" +exit 0 +` + + sleepShim = `#!/bin/sh +printf 'sleep %s\n' "$*" >> "$STUB_LOG" +exit 0 +` + + tarShim = `#!/bin/sh +printf 'tar %s\n' "$*" >> "$STUB_LOG" +[ "${STUB_TAR:-0}" = 0 ] || exit "$STUB_TAR" +mode= +file= +dir=. +prev= +for arg in "$@"; do +case "$arg" in +-cf) mode=c;; +-xf) mode=x;; +esac +case "$prev" in +-C) dir=$arg;; +-cf|-xf) file=$arg;; +esac +prev=$arg +done +case "$mode" in +c) printf 'archive\n' > "$file";; +x) [ -z "$STUB_TAR_MEMBER" ] || printf 'member\n' > "$dir/$STUB_TAR_MEMBER";; +esac +exit 0 +` + + orasShim = `#!/bin/sh +printf 'oras %s\n' "$*" >> "$STUB_LOG" +case "$1 $2" in +"pull "*) +[ "${STUB_ORAS_PULL:-0}" = 0 ] || exit "$STUB_ORAS_PULL" +out= +prev= +for arg in "$@"; do +if [ "$prev" = "-o" ]; then out=$arg; fi +prev=$arg +done +for name in $STUB_ORAS_FILES; do +printf 'blob\n' > "$out/$name" +done +;; +"manifest fetch") +[ "${STUB_ORAS_FETCH:-0}" = 0 ] || exit "$STUB_ORAS_FETCH" +printf '%s\n' "$STUB_DESCRIPTOR" +;; +"push "*) +[ "${STUB_ORAS_PUSH:-0}" = 0 ] || exit "$STUB_ORAS_PUSH" +;; +esac +exit 0 +` + + journalctlShim = `#!/bin/sh +printf 'journalctl %s\n' "$*" >> "$STUB_LOG" +printf '%s\n' "$STUB_JOURNAL" +[ "${STUB_JOURNAL_HOLD:-0}" = 1 ] || exit 0 +trap 'printf "journalctl killed\n" >> "$STUB_LOG"; exit 0' TERM +/bin/sleep 5 >/dev/null 2>&1 & +wait +exit 0 +` +) + +func TestMetaScriptReportsTheRecordAndTheMountState(t *testing.T) { + tests := []struct { + name string + meta string + mountpoint string + wantCode int + wantStdout string + }{ + {"a mounted overlay", overlayMeta, "0", 0, "1\n" + overlayMeta + "\n"}, + {"an unmounted overlay", overlayMeta, "1", 0, "0\n" + overlayMeta + "\n"}, + {"a raw workload", rawMeta, "1", 0, "0\n" + rawMeta + "\n"}, + {"a workload the node lost", "", "1", workloadmeta.NotExistsCode, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_MOUNTPOINT"] = tt.mountpoint + if tt.meta != "" { + node.write(t, filepath.Join(node.dir, "meta.json"), tt.meta+"\n") + } + + got := node.run(t, metaScript, node.dir) + + if got.code != tt.wantCode { + t.Errorf("got exit %d, want %d", got.code, tt.wantCode) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + }) + } +} + +func TestStartScriptPreparesTheOverlayBeforeTheUnit(t *testing.T) { + tests := []struct { + name string + loadState string + subState string + mountpoint string + work bool + fail string + wantCode int + wantCalls []string + wantRun bool + }{ + { + name: "a running unit is left alone", + subState: subStateRunning, + }, + { + name: "a stale unit is stopped and its overlay mounted", + loadState: "loaded", + subState: "dead", + mountpoint: "1", + work: true, + wantCalls: []string{ + "systemctl stop " + scriptUnit, + "mountpoint -q {dir}/merged", + "mount -t overlay overlay -o lowerdir={dir}/lower,upperdir={dir}/upper,workdir={dir}/work {dir}/merged", + }, + wantRun: true, + }, + { + name: "a raw workload has no overlay to mount", + loadState: "not-found", + subState: "dead", + wantRun: true, + }, + { + name: "an overlay that is already mounted is kept", + loadState: "not-found", + subState: "dead", + mountpoint: "0", + work: true, + wantCalls: []string{"mountpoint -q {dir}/merged"}, + wantRun: true, + }, + { + name: "a unit that will not stop fails the start", + loadState: "loaded", + subState: "dead", + fail: "stop", + wantCode: 1, + wantCalls: []string{"systemctl stop " + scriptUnit}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_LOADSTATE"] = tt.loadState + node.env["STUB_MOUNTPOINT"] = tt.mountpoint + node.env["STUB_FAIL"] = tt.fail + node.states(t, tt.subState) + node.write(t, filepath.Join(node.dir, "meta.json"), overlayMeta) + node.write(t, filepath.Join(node.dir, "run.sh"), "echo started\n") + if tt.work { + node.mkdir(t, filepath.Join(node.dir, "work")) + } + + got := node.run(t, startScript, node.dir, scriptUnit, node.record) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + node.assertCalls(t, tt.wantCalls...) + if !tt.wantRun { + if got.stdout != "" { + t.Errorf("got %q, want the unit left unstarted", got.stdout) + } + return + } + if got.stdout != "started\n" { + t.Errorf("got %q, want the launcher exec'd", got.stdout) + } + if body := node.read(t, node.record); body != overlayMeta { + t.Errorf("got %q, want the record published for eru-agent", body) + } + }) + } +} + +func TestStopScriptStopsTheUnitAndDropsTheOverlay(t *testing.T) { + tests := []struct { + name string + loadState string + mountpoint string + force string + fail string + wantCode int + wantCalls []string + }{ + { + name: "a forced stop kills first", + loadState: "loaded", + mountpoint: "1", + force: "1", + wantCalls: []string{ + "systemctl kill -s SIGKILL " + scriptUnit, + "systemctl stop " + scriptUnit, + "mountpoint -q {dir}/merged", + }, + }, + { + name: "a graceful stop never kills", + loadState: "loaded", + mountpoint: "1", + force: "0", + wantCalls: []string{ + "systemctl stop " + scriptUnit, + "mountpoint -q {dir}/merged", + }, + }, + { + name: "an unloaded unit is only unmounted", + loadState: "not-found", + mountpoint: "0", + force: "1", + wantCalls: []string{ + "mountpoint -q {dir}/merged", + "umount -l {dir}/merged", + }, + }, + { + name: "a kill on a unit systemd already dropped is tolerated", + loadState: "loaded", + mountpoint: "1", + force: "1", + fail: "kill", + wantCalls: []string{ + "systemctl kill -s SIGKILL " + scriptUnit, + "systemctl stop " + scriptUnit, + "mountpoint -q {dir}/merged", + }, + }, + { + name: "a failed stop is reported", + loadState: "loaded", + mountpoint: "1", + force: "1", + fail: "stop", + wantCode: 1, + wantCalls: []string{ + "systemctl kill -s SIGKILL " + scriptUnit, + "systemctl stop " + scriptUnit, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_LOADSTATE"] = tt.loadState + node.env["STUB_MOUNTPOINT"] = tt.mountpoint + node.env["STUB_FAIL"] = tt.fail + + got := node.run(t, stopScript, scriptUnit, node.dir, tt.force) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + node.assertCalls(t, tt.wantCalls...) + }) + } +} + +func TestRemoveScriptUnmountsTheOverlayItDrops(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_LOADSTATE"] = "loaded" + node.env["STUB_MOUNTPOINT"] = "0" + node.states(t, "dead") + node.write(t, node.record, overlayMeta) + + got := node.run(t, removeScript, scriptUnit, node.dir, node.record, "0") + + if got.code != 0 { + t.Fatalf("got exit %d, want a stopped workload removed: %s", got.code, got.stderr) + } + node.assertCalls(t, + "systemctl reset-failed "+scriptUnit, + "mountpoint -q {dir}/merged", + "umount -l {dir}/merged", + ) + for _, path := range []string{node.dir, node.record} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s survived the remove: %v", path, err) + } + } +} + +func TestInspectScriptShowsTheUnitOfALiveWorkload(t *testing.T) { + tests := []struct { + name string + keepDir bool + wantCode int + wantStdout string + }{ + {"a workload the node still has", true, 0, showOutput}, + {"a workload the node lost", false, workloadmeta.NotExistsCode, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_SHOW"] = showOutput + if !tt.keepDir { + node.remove(t, node.dir) + } + + got := node.run(t, inspectScript, node.dir, scriptUnit) + + if got.code != tt.wantCode { + t.Errorf("got exit %d, want %d", got.code, tt.wantCode) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + }) + } +} + +func TestWaitScriptPollsUntilTheUnitLeavesRunning(t *testing.T) { + tests := []struct { + name string + states []string + status string + wantSleeps int + }{ + {"a running unit is polled", []string{subStateRunning, subStateRunning, "exited"}, "3", 2}, + {"a dead unit answers at once", []string{"dead"}, "0", 0}, + {"a failed unit answers at once", []string{"failed"}, "1", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.states(t, tt.states...) + node.env["STUB_STATUS"] = tt.status + + got := node.run(t, waitScript, scriptUnit) + + if got.code != 0 { + t.Fatalf("got exit %d, want the wait to end cleanly: %s", got.code, got.stderr) + } + if got.stdout != tt.status+"\n" { + t.Errorf("got %q, want ExecMainStatus %q", got.stdout, tt.status) + } + if sleeps := len(node.calls(t)); sleeps != tt.wantSleeps { + t.Errorf("got %d polls, want %d", sleeps, tt.wantSleeps) + } + }) + } +} + +func TestListScriptPrintsTheImageCache(t *testing.T) { + tests := []struct { + name string + entries []string + wantStdout string + }{ + {"a cache with images", []string{"a%2Fb", "c%2Fd"}, "a%2Fb\nc%2Fd\n"}, + {"a node that never pulled", nil, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + cache := filepath.Join(node.root, imageCache) + for _, entry := range tt.entries { + node.mkdir(t, filepath.Join(cache, entry)) + } + + got := node.run(t, listScript, cache) + + if got.code != 0 { + t.Fatalf("got exit %d, want a missing cache reported as empty: %s", got.code, got.stderr) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + }) + } +} + +func TestPullScriptUnpacksTheBundleAndRecordsTheDigest(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_FILES"] = "bundle.tar" + node.env["STUB_DESCRIPTOR"] = scriptDescriptor + node.env["STUB_TAR_MEMBER"] = "server" + cache := filepath.Join(node.root, imageCache, "app") + node.write(t, filepath.Join(cache, "stale"), "old") + + got := node.run(t, pullScript, scriptRef, cache, "--plain-http") + + if got.code != 0 { + t.Fatalf("got exit %d, want the pull to land: %s", got.code, got.stderr) + } + node.assertCalls(t, + "oras pull "+scriptRef+" -o "+cache+" --plain-http", + "tar -C "+cache+" -xf "+cache+"/bundle.tar", + "oras manifest fetch --descriptor "+scriptRef+" --plain-http", + ) + if body := node.read(t, filepath.Join(cache, digestFile)); body != scriptDescriptor+"\n" { + t.Errorf("got %q, want the descriptor cached", body) + } + for _, path := range []string{filepath.Join(cache, "bundle.tar"), filepath.Join(cache, "stale")} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s survived the pull: %v", path, err) + } + } + if _, err := os.Stat(filepath.Join(cache, "server")); err != nil { + t.Errorf("the unpacked bundle is missing: %v", err) + } +} + +func TestPullScriptLeavesAnUnpackedArtifactAlone(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_FILES"] = "server" + node.env["STUB_DESCRIPTOR"] = scriptDescriptor + cache := filepath.Join(node.root, imageCache, "app") + + got := node.run(t, pullScript, scriptRef, cache) + + if got.code != 0 { + t.Fatalf("got exit %d, want the pull to land: %s", got.code, got.stderr) + } + node.assertCalls(t, + "oras pull "+scriptRef+" -o "+cache, + "oras manifest fetch --descriptor "+scriptRef, + ) +} + +func TestPullScriptRecordsNoDigestWhenTheRegistryRefuses(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_PULL"] = "1" + cache := filepath.Join(node.root, imageCache, "app") + + got := node.run(t, pullScript, scriptRef, cache) + + if got.code == 0 { + t.Fatal("got exit 0, want a failed pull reported") + } + if _, err := os.Stat(filepath.Join(cache, digestFile)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a failed pull must leave no digest: %v", err) + } +} + +func TestCreateScriptPullsTheBundleAndPublishesTheRecord(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_FILES"] = "bundle.tar" + node.env["STUB_TAR_MEMBER"] = "server" + bind := filepath.Join(node.root, "data") + + got := node.create(t, "1", filepath.Join(node.root, "cache"), bind, "--plain-http") + + if got.code != 0 { + t.Fatalf("got exit %d, want the workload created: %s", got.code, got.stderr) + } + node.assertCalls(t, + "oras pull "+scriptRef+" -o {dir}/lower --plain-http", + "tar -C {dir}/lower -xf {dir}/lower/bundle.tar", + ) + for _, path := range []string{"lower/server", "upper", "work", "merged"} { + if _, err := os.Stat(filepath.Join(node.dir, path)); err != nil { + t.Errorf("%s is missing: %v", path, err) + } + } + if _, err := os.Stat(bind); err != nil { + t.Errorf("the bind source was not created: %v", err) + } + for path, want := range map[string]string{ + filepath.Join(node.dir, "run.sh"): scriptLauncher + "\n", + filepath.Join(node.dir, propsFile): "CPUQuota=200%\n", + filepath.Join(node.dir, "meta.json"): overlayMeta + "\n", + node.record: overlayMeta + "\n", + } { + if body := node.read(t, path); body != want { + t.Errorf("got %q in %s, want %q", body, path, want) + } + } +} + +func TestCreateScriptSeedsTheLowerDirFromTheImageCache(t *testing.T) { + node := newScriptNode(t) + cache := filepath.Join(node.root, "cache") + node.write(t, filepath.Join(cache, "server"), "binary") + node.write(t, filepath.Join(cache, digestFile), scriptDescriptor) + + got := node.create(t, "1", cache, "") + + if got.code != 0 { + t.Fatalf("got exit %d, want the cached bundle copied: %s", got.code, got.stderr) + } + node.assertCalls(t) + if body := node.read(t, filepath.Join(node.dir, "lower", "server")); body != "binary" { + t.Errorf("got %q, want the cached bundle in place", body) + } + if _, err := os.Stat(filepath.Join(node.dir, "lower", digestFile)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the digest of the cache must not reach the workload: %v", err) + } +} + +func TestCreateScriptGivesARawWorkloadNoOverlay(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_FILES"] = "server" + + got := node.create(t, "0", filepath.Join(node.root, "cache"), "") + + if got.code != 0 { + t.Fatalf("got exit %d, want the workload created: %s", got.code, got.stderr) + } + for _, path := range []string{"upper", "work", "merged"} { + if _, err := os.Stat(filepath.Join(node.dir, path)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a raw workload must not get %s: %v", path, err) + } + } +} + +func TestCreateScriptDropsTheWorkloadDirWhenThePullFails(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_ORAS_PULL"] = "1" + + got := node.create(t, "1", filepath.Join(node.root, "cache"), "") + + if got.code == 0 { + t.Fatal("got exit 0, want a failed pull reported") + } + for _, path := range []string{node.dir, node.record} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s survived a failed create: %v", path, err) + } + } +} + +func TestExistScriptCapturesTheFrozenOverlay(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_MOUNTPOINT"] = "1" + node.env["STUB_DESCRIPTOR"] = scriptDescriptor + layer := filepath.Join(node.dir, existArchive) + + got := node.run(t, existScript, scriptUnit, node.dir, scriptRef, layer, "--plain-http") + + if got.code != 0 { + t.Fatalf("got exit %d, want the bundle pushed: %s", got.code, got.stderr) + } + if got.stdout != scriptDescriptor+"\n" { + t.Errorf("got %q, want the descriptor of the pushed artifact", got.stdout) + } + node.assertCalls(t, + "mountpoint -q {dir}/merged", + "mount -t overlay overlay -o lowerdir={dir}/lower,upperdir={dir}/upper,workdir={dir}/work {dir}/merged", + "systemctl freeze "+scriptUnit, + "tar -C {dir}/merged -cf "+layer+" .", + "systemctl thaw "+scriptUnit, + "oras push --disable-path-validation --artifact-type "+bundleMedia+" "+scriptRef+" "+layer+":"+bundleMedia+" --plain-http", + "oras manifest fetch --descriptor "+scriptRef+" --plain-http", + "systemctl thaw "+scriptUnit, + "umount -l {dir}/merged", + ) + if _, err := os.Stat(layer); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the layer archive survived the push: %v", err) + } +} + +func TestExistScriptKeepsAnOverlayItDidNotMount(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_MOUNTPOINT"] = "0" + node.env["STUB_DESCRIPTOR"] = scriptDescriptor + layer := filepath.Join(node.dir, existArchive) + + got := node.run(t, existScript, scriptUnit, node.dir, scriptRef, layer) + + if got.code != 0 { + t.Fatalf("got exit %d, want the bundle pushed: %s", got.code, got.stderr) + } + for _, call := range node.calls(t) { + if strings.HasPrefix(call, "mount ") || strings.HasPrefix(call, "umount ") { + t.Errorf("got %q, want the running workload's own mount left alone", call) + } + } +} + +func TestExistScriptThawsTheUnitWhenTheCaptureFails(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_MOUNTPOINT"] = "1" + node.env["STUB_TAR"] = "2" + layer := filepath.Join(node.dir, existArchive) + + got := node.run(t, existScript, scriptUnit, node.dir, scriptRef, layer) + + if got.code != 2 { + t.Fatalf("got exit %d, want the tar failure reported", got.code) + } + node.assertCalls(t, + "mountpoint -q {dir}/merged", + "mount -t overlay overlay -o lowerdir={dir}/lower,upperdir={dir}/upper,workdir={dir}/work {dir}/merged", + "systemctl freeze "+scriptUnit, + "tar -C {dir}/merged -cf "+layer+" .", + "systemctl thaw "+scriptUnit, + "umount -l {dir}/merged", + ) +} + +func TestFollowScriptEndsTheJournalWithTheUnit(t *testing.T) { + node := newScriptNode(t) + node.states(t, subStateRunning, "dead") + node.env["STUB_JOURNAL"] = "boot line" + node.env["STUB_JOURNAL_HOLD"] = "1" + + got := node.run(t, followScript, scriptUnit, "-f", "-n", "10") + + if got.code != 0 { + t.Fatalf("got exit %d, want the follow to end cleanly: %s", got.code, got.stderr) + } + if !strings.Contains(got.stdout, "boot line") { + t.Errorf("got %q, want the journal streamed", got.stdout) + } + node.assertHasCalls(t, "journalctl -u "+scriptUnit+" -f -n 10", "sleep 1", "journalctl killed") +} + +func TestChdirScriptRunsTheCommandInTheWorkingDir(t *testing.T) { + node := newScriptNode(t) + + got := node.run(t, chdirScript, node.dir, "pwd") + + if got.code != 0 { + t.Fatalf("got exit %d, want the command run: %s", got.code, got.stderr) + } + if strings.TrimSpace(got.stdout) != node.dir { + t.Errorf("got %q, want the command run in %q", got.stdout, node.dir) + } +} + +func TestChdirScriptFailsOnAWorkingDirThatIsGone(t *testing.T) { + node := newScriptNode(t) + + got := node.run(t, chdirScript, filepath.Join(node.root, "gone"), "pwd") + + if got.code == 0 { + t.Fatal("got exit 0, want an unusable working dir reported") + } + if got.stdout != "" { + t.Errorf("got %q, want the command left unrun", got.stdout) + } +} + +type scriptRun struct { + code int + stdout string + stderr string +} + +type scriptNode struct { + root string + bin string + dir string + record string + logPath string + path string + env map[string]string +} + +func newScriptNode(t *testing.T) *scriptNode { + t.Helper() + root := t.TempDir() + node := &scriptNode{ + root: root, + bin: filepath.Join(root, "bin"), + dir: filepath.Join(root, "workloads", "w1"), + record: filepath.Join(root, "run", "w1.json"), + logPath: filepath.Join(root, "calls.log"), + } + node.path = node.bin + ":/usr/bin:/bin" + node.env = map[string]string{ + "STUB_LOG": node.logPath, + "STUB_SUBSTATE_FILE": filepath.Join(root, "substate"), + "STUB_LOADSTATE": "not-found", + } + node.mkdir(t, node.dir) + for name, body := range map[string]string{ + "systemctl": systemctlShim, + "mountpoint": mountpointShim, + "mount": mountShim, + "umount": umountShim, + "sleep": sleepShim, + "tar": tarShim, + "oras": orasShim, + "journalctl": journalctlShim, + } { + node.write(t, filepath.Join(node.bin, name), body) + if err := os.Chmod(filepath.Join(node.bin, name), 0o755); err != nil { + t.Fatalf("setup %s: %v", name, err) + } + } + node.states(t, "dead") + return node +} + +func (n *scriptNode) run(t *testing.T, script string, args ...string) scriptRun { + t.Helper() + cmd := exec.CommandContext(t.Context(), "/bin/sh", slices.Concat([]string{"-c", script, "sh"}, args)...) + cmd.Env = slices.Concat([]string{"PATH=" + n.path, "HOME=" + n.root, "TMPDIR=" + os.TempDir()}, n.environ()) + stdout, stderr := &strings.Builder{}, &strings.Builder{} + cmd.Stdout, cmd.Stderr = stdout, stderr + err := cmd.Run() + got := scriptRun{stdout: stdout.String(), stderr: stderr.String()} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + got.code = exitErr.ExitCode() + default: + t.Fatalf("run: %v", err) + } + return got +} + +func (n *scriptNode) create(t *testing.T, overlay, cache, bind string, flags ...string) scriptRun { + t.Helper() + args := slices.Concat([]string{ + n.dir, scriptRef, cache, scriptLauncher, n.record, overlay, overlayMeta, bind, "CPUQuota=200%", + }, flags) + return n.run(t, createScript, args...) +} + +func (n *scriptNode) environ() []string { + env := make([]string, 0, len(n.env)) + for key, value := range n.env { + env = append(env, key+"="+value) + } + return env +} + +func (n *scriptNode) states(t *testing.T, states ...string) { + t.Helper() + n.write(t, n.env["STUB_SUBSTATE_FILE"], strings.Join(states, "\n")+"\n") +} + +func (n *scriptNode) calls(t *testing.T) []string { + t.Helper() + body, err := os.ReadFile(n.logPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + t.Fatalf("read log: %v", err) + } + return strings.Split(strings.TrimSuffix(string(body), "\n"), "\n") +} + +func (n *scriptNode) assertCalls(t *testing.T, want ...string) { + t.Helper() + got := n.calls(t) + expanded := make([]string, 0, len(want)) + for _, line := range want { + expanded = append(expanded, n.expand(line)) + } + if !slices.Equal(got, expanded) { + t.Errorf("got calls %q, want %q", got, expanded) + } +} + +func (n *scriptNode) assertHasCalls(t *testing.T, want ...string) { + t.Helper() + got := n.calls(t) + for _, line := range want { + if !slices.Contains(got, n.expand(line)) { + t.Errorf("got calls %q, want one of them to be %q", got, n.expand(line)) + } + } +} + +func (n *scriptNode) expand(line string) string { + return strings.ReplaceAll(line, "{dir}", n.dir) +} + +func (n *scriptNode) write(t *testing.T, path, body string) { + t.Helper() + n.mkdir(t, filepath.Dir(path)) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("setup %s: %v", path, err) + } +} + +func (n *scriptNode) read(t *testing.T, path string) string { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(body) +} + +func (n *scriptNode) mkdir(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("setup %s: %v", path, err) + } +} + +func (n *scriptNode) remove(t *testing.T, path string) { + t.Helper() + if err := os.RemoveAll(path); err != nil { + t.Fatalf("setup %s: %v", path, err) + } +}