diff --git a/free/ci/cmd/logs.go b/free/ci/cmd/logs.go index eba615fe9..b624172cd 100644 --- a/free/ci/cmd/logs.go +++ b/free/ci/cmd/logs.go @@ -103,7 +103,8 @@ func runLogs(args []string) int { } // attemptEnded reports whether the attempt can no longer write output. Without -// a store (or without a state for it) nothing is running. +// a store (or without a state for it) nothing is running. A retryable attempt is +// parked until the coordinator queues it again, so it ends the follow too. func attemptEnded(ctx context.Context, st *store.Store, attempt string) bool { if st == nil { return true @@ -113,7 +114,7 @@ func attemptEnded(ctx context.Context, st *store.Store, attempt string) bool { return true } switch state { - case model.JobState("passed"), "failed", "cancelled", "lost", "needs-operator": + case model.JobState("passed"), "failed", "cancelled", "lost", "retryable", "needs-operator": return true } return false diff --git a/free/ci/cmd/logs_ended_test.go b/free/ci/cmd/logs_ended_test.go new file mode 100644 index 000000000..ff822f820 --- /dev/null +++ b/free/ci/cmd/logs_ended_test.go @@ -0,0 +1,62 @@ +package main + +// D-1188: `ci logs -f` stops following when the attempt cannot write output any +// more. A retryable attempt is parked until the coordinator queues it again, so +// after a coordinator crash it must end the follow, as lost and failed do. + +import ( + "context" + "path/filepath" + "testing" + + "github.com/nself-org/plugins/free/ci/internal/model" + "github.com/nself-org/plugins/free/ci/internal/store" + "github.com/nself-org/plugins/free/ci/internal/storetest" +) + +func TestAttemptEndedForParkedAndLiveStates(t *testing.T) { + ctx := context.Background() + st, err := store.Open(filepath.Join(t.TempDir(), "state.db"), store.Options{}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = st.Close() }() + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(st.CreatePipeline(ctx, store.PipelineRow{ID: "run-1", Revision: sealRev, Project: "proj", Trigger: "local", SourceTrust: "owner", + PrivacyZone: "local-only", PolicyDigest: "p", InputDigest: "in1", SelectionMode: "full", Status: "running"})) + must(st.CreateJob(ctx, store.JobRow{ID: "run-1.test-1", PipelineID: "run-1", Name: "test", Kind: "test", Idempotent: true, InfraMax: 1, InputDigest: "i"})) + const attempt = "run-1.test-1.a1" + must(st.CreateAttempt(ctx, store.AttemptRow{ID: attempt, JobID: "run-1.test-1", N: 1}, "")) + + step := func(from, to model.JobState) { + t.Helper() + _, epoch, err := st.AttemptStateEpoch(ctx, attempt) + must(err) + if to == "leased" { + must(storetest.Admit(ctx, st, attempt)) + return + } + must(st.Transition(ctx, attempt, from, to, store.TransitionDetail{Epoch: epoch, FailureClass: "infra"})) + } + if attemptEnded(ctx, st, attempt) { + t.Fatal("a queued attempt can still write output") + } + step("queued", "leased") + step("leased", "running") + if attemptEnded(ctx, st, attempt) { + t.Fatal("a running attempt can still write output") + } + step("running", "lost") + if !attemptEnded(ctx, st, attempt) { + t.Fatal("a lost attempt cannot write output") + } + step("lost", "retryable") + if !attemptEnded(ctx, st, attempt) { + t.Fatal("a retryable attempt is parked; following it would wait forever after a coordinator crash") + } +} diff --git a/free/ci/internal/exec/container.go b/free/ci/internal/exec/container.go index 05afbaabe..8d8208cb6 100644 --- a/free/ci/internal/exec/container.go +++ b/free/ci/internal/exec/container.go @@ -25,6 +25,17 @@ func dockerHostEnv() []string { return env } +// containerRuntime is the container CLI of this host: docker when installed, else +// podman. The toolchain resolver makes the same choice for the image pull. +func containerRuntime() string { + for _, bin := range []string{"docker", "podman"} { + if _, err := osexec.LookPath(bin); err == nil { + return bin + } + } + return "docker" +} + // containerWorkdir maps the job workdir to its path under the /repo mount. func containerWorkdir(s JobSpec) (string, error) { if s.Job.Workdir == "" || s.Job.Workdir == "." { @@ -82,8 +93,9 @@ func (e *Executor) runContainer(ctx context.Context, s JobSpec, workspace string } args = append(args, s.Job.Container.Image) args = append(args, s.Command...) - stop := containerStopper("docker", name, e.Grace) - code, err := e.runCommandIn(ctx, s.Stdin, append([]string{"docker"}, args...), workspace, cliEnv, s.Output, stop, nil, "") + runtime := containerRuntime() + stop := containerStopper(runtime, name, e.Grace) + code, err := e.runCommandIn(ctx, s.Stdin, append([]string{runtime}, args...), workspace, cliEnv, s.Output, stop, nil, "") if err != nil { stop() return code, fmt.Errorf("%w: %v", coded("E613", "docker run"), err) diff --git a/free/ci/internal/exec/container_env_test.go b/free/ci/internal/exec/container_env_test.go index c9439de0f..7a15f289d 100644 --- a/free/ci/internal/exec/container_env_test.go +++ b/free/ci/internal/exec/container_env_test.go @@ -90,3 +90,21 @@ func TestBaselineContainerKeepsImagePathAndToolchain(t *testing.T) { } } } + +// D-1185: a host with podman and no docker runs the baseline container with podman. +func TestBaselineContainerUsesPodmanWhenDockerIsAbsent(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "podman-ran") + if err := os.WriteFile(filepath.Join(dir, "podman"), []byte("#!/bin/sh\n: > "+marker+"\nexit 0\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + r := (&Executor{Env: AllowlistEnv{}}).Run(context.Background(), JobSpec{AttemptID: "podman", CoordinatorID: "coord", Home: t.TempDir(), + Job: model.Job{Isolation: "container", Container: &model.ContainerConfig{Image: "alpine:3.19"}}, Command: []string{"true"}}) + if r.Err != nil { + t.Fatal(r.Err) + } + if _, err := os.Stat(marker); err != nil { + t.Fatalf("podman was not run: %v", err) + } +} diff --git a/free/ci/internal/toolchain/image.go b/free/ci/internal/toolchain/image.go index db0cf6fa7..36255b435 100644 --- a/free/ci/internal/toolchain/image.go +++ b/free/ci/internal/toolchain/image.go @@ -31,17 +31,23 @@ var refRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._/:-]*@sha256:[0-9a-f]{64}$`) // only when the process could not run (the doctor and report funnel). type Capture func(ctx context.Context, argv ...string) (string, int, error) +// rcNotFound is the exit code of the shell when the command is not installed. +const rcNotFound = 127 + // Resolver turns the environment and the pins into an image reference. type Resolver struct { Getenv func(string) string ReadFile func(string) ([]byte, error) Capture Capture // nil: do not check or pull the image + // Runtimes are the container CLIs tried in order; the first one that can run + // is used for the inspect and the pull. Empty means docker, then podman. + Runtimes []string mu sync.Mutex have map[string]bool } -// NewResolver reads the real process environment and uses capture for docker. +// NewResolver reads the real process environment and uses capture for docker or podman. func NewResolver(capture Capture) *Resolver { return &Resolver{Getenv: os.Getenv, ReadFile: os.ReadFile, Capture: capture} } @@ -95,7 +101,7 @@ func (r *Resolver) table() (Versions, error) { return ParseVersions(b) } -// ensure makes ref present on the host. docker verifies the manifest digest +// ensure makes ref present on the host. The container runtime verifies the manifest digest // while pulling, so content that does not match the pin never lands. func (r *Resolver) ensure(ctx context.Context, ref string) error { if r.Capture == nil { @@ -106,10 +112,25 @@ func (r *Resolver) ensure(ctx context.Context, ref string) error { if r.have[ref] { return nil } - if _, rc, err := r.Capture(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", ref); err == nil && rc == 0 { - return r.remember(ref) + runtimes := r.Runtimes + if len(runtimes) == 0 { + runtimes = []string{"docker", "podman"} + } + for i, rt := range runtimes { + _, rc, err := r.Capture(ctx, rt, "image", "inspect", "--format", "{{.Id}}", ref) + if (err != nil || rc == rcNotFound) && i < len(runtimes)-1 { + continue // this CLI is not installed: try the next one + } + if err == nil && rc == 0 { + return r.remember(ref) + } + return r.pull(ctx, rt, ref) } - out, rc, err := r.Capture(ctx, "docker", "pull", "--quiet", ref) + return nil // unreachable: the last runtime always returns above +} + +func (r *Resolver) pull(ctx context.Context, runtime, ref string) error { + out, rc, err := r.Capture(ctx, runtime, "pull", "--quiet", ref) if err != nil || rc != 0 { return fail("cannot pull %s (digest mismatch, unknown digest or registry unreachable): %s", ref, tail(out, err)) } diff --git a/free/ci/internal/toolchain/image_test.go b/free/ci/internal/toolchain/image_test.go index ca71c62d3..47d083c3c 100644 --- a/free/ci/internal/toolchain/image_test.go +++ b/free/ci/internal/toolchain/image_test.go @@ -105,3 +105,38 @@ func TestEnsurePullsOnceAndMapsFailureToE645(t *testing.T) { t.Fatalf("failed pull error = %v", err) } } + +// D-1185: with no docker CLI the image is inspected and pulled with podman. +func TestEnsureFallsBackToPodmanWhenDockerCannotRun(t *testing.T) { + ref := "docker.io/library/alpine:3.22@" + pinnedDigest + env := envOf(map[string]string{EnvImage: ref}) + var calls []string + capture := func(_ context.Context, argv ...string) (string, int, error) { + calls = append(calls, argv[0]+" "+argv[1]) + if argv[0] == "docker" { // the exec funnel reports a missing CLI as the shell's 127 + return "ci-job: line 0: exec: docker: not found", 127, nil + } + if argv[1] == "image" { + return "", 1, nil + } + return "pulled", 0, nil + } + if got, err := (&Resolver{Getenv: env, Capture: capture}).Resolve(context.Background()); err != nil || got != ref { + t.Fatalf("resolve: %q %v", got, err) + } + if want := "docker image,podman image,podman pull"; strings.Join(calls, ",") != want { + t.Fatalf("calls %v, want %s", calls, want) + } + + calls = nil + present := func(_ context.Context, argv ...string) (string, int, error) { + calls = append(calls, argv[0]+" "+argv[1]) + if argv[0] == "docker" { + return "", -1, errors.New("executable file not found") + } + return "", 0, nil + } + if _, err := (&Resolver{Getenv: env, Capture: present}).Resolve(context.Background()); err != nil || len(calls) != 2 { + t.Fatalf("image present under podman: %v calls %v", err, calls) + } +}