fix(ci): podman image pull and ci logs -f on retryable attempts (D-1185, D-1188)

free/ci/cmd/logs.go

@@ -103,7 +103,8 @@ func runLogs(args []string) int {
 }
 
 // attemptEnded reports whether the attempt can no longer write output. Without
-// a store (or without a state for it) nothing is running.
+// a store (or without a state for it) nothing is running. A retryable attempt is
+// parked until the coordinator queues it again, so it ends the follow too.
 func attemptEnded(ctx context.Context, st *store.Store, attempt string) bool {
 	if st == nil {
 		return true
@@ -113,7 +114,7 @@ func attemptEnded(ctx context.Context, st *store.Store, attempt string) bool {
 		return true
 	}
 	switch state {
-	case model.JobState("passed"), "failed", "cancelled", "lost", "needs-operator":
+	case model.JobState("passed"), "failed", "cancelled", "lost", "retryable", "needs-operator":
 		return true
 	}
 	return false

free/ci/cmd/logs_ended_test.go

@@ -0,0 +1,62 @@
+package main
+
+// D-1188: `ci logs -f` stops following when the attempt cannot write output any
+// more. A retryable attempt is parked until the coordinator queues it again, so
+// after a coordinator crash it must end the follow, as lost and failed do.
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/nself-org/plugins/free/ci/internal/model"
+	"github.com/nself-org/plugins/free/ci/internal/store"
+	"github.com/nself-org/plugins/free/ci/internal/storetest"
+)
+
+func TestAttemptEndedForParkedAndLiveStates(t *testing.T) {
+	ctx := context.Background()
+	st, err := store.Open(filepath.Join(t.TempDir(), "state.db"), store.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = st.Close() }()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	must(st.CreatePipeline(ctx, store.PipelineRow{ID: "run-1", Revision: sealRev, Project: "proj", Trigger: "local", SourceTrust: "owner",
+		PrivacyZone: "local-only", PolicyDigest: "p", InputDigest: "in1", SelectionMode: "full", Status: "running"}))
+	must(st.CreateJob(ctx, store.JobRow{ID: "run-1.test-1", PipelineID: "run-1", Name: "test", Kind: "test", Idempotent: true, InfraMax: 1, InputDigest: "i"}))
+	const attempt = "run-1.test-1.a1"
+	must(st.CreateAttempt(ctx, store.AttemptRow{ID: attempt, JobID: "run-1.test-1", N: 1}, ""))
+
+	step := func(from, to model.JobState) {
+		t.Helper()
+		_, epoch, err := st.AttemptStateEpoch(ctx, attempt)
+		must(err)
+		if to == "leased" {
+			must(storetest.Admit(ctx, st, attempt))
+			return
+		}
+		must(st.Transition(ctx, attempt, from, to, store.TransitionDetail{Epoch: epoch, FailureClass: "infra"}))
+	}
+	if attemptEnded(ctx, st, attempt) {
+		t.Fatal("a queued attempt can still write output")
+	}
+	step("queued", "leased")
+	step("leased", "running")
+	if attemptEnded(ctx, st, attempt) {
+		t.Fatal("a running attempt can still write output")
+	}
+	step("running", "lost")
+	if !attemptEnded(ctx, st, attempt) {
+		t.Fatal("a lost attempt cannot write output")
+	}
+	step("lost", "retryable")
+	if !attemptEnded(ctx, st, attempt) {
+		t.Fatal("a retryable attempt is parked; following it would wait forever after a coordinator crash")
+	}
+}

free/ci/internal/exec/container.go

@@ -25,6 +25,17 @@ func dockerHostEnv() []string {
 	return env
 }
 
+// containerRuntime is the container CLI of this host: docker when installed, else
+// podman. The toolchain resolver makes the same choice for the image pull.
+func containerRuntime() string {
+	for _, bin := range []string{"docker", "podman"} {
+		if _, err := osexec.LookPath(bin); err == nil {
+			return bin
+		}
+	}
+	return "docker"
+}
+
 // containerWorkdir maps the job workdir to its path under the /repo mount.
 func containerWorkdir(s JobSpec) (string, error) {
 	if s.Job.Workdir == "" || s.Job.Workdir == "." {
@@ -82,8 +93,9 @@ func (e *Executor) runContainer(ctx context.Context, s JobSpec, workspace string
 	}
 	args = append(args, s.Job.Container.Image)
 	args = append(args, s.Command...)
-	stop := containerStopper("docker", name, e.Grace)
-	code, err := e.runCommandIn(ctx, s.Stdin, append([]string{"docker"}, args...), workspace, cliEnv, s.Output, stop, nil, "")
+	runtime := containerRuntime()
+	stop := containerStopper(runtime, name, e.Grace)
+	code, err := e.runCommandIn(ctx, s.Stdin, append([]string{runtime}, args...), workspace, cliEnv, s.Output, stop, nil, "")
 	if err != nil {
 		stop()
 		return code, fmt.Errorf("%w: %v", coded("E613", "docker run"), err)

free/ci/internal/exec/container_env_test.go

@@ -90,3 +90,21 @@ func TestBaselineContainerKeepsImagePathAndToolchain(t *testing.T) {
 		}
 	}
 }
+
+// D-1185: a host with podman and no docker runs the baseline container with podman.
+func TestBaselineContainerUsesPodmanWhenDockerIsAbsent(t *testing.T) {
+	dir := t.TempDir()
+	marker := filepath.Join(dir, "podman-ran")
+	if err := os.WriteFile(filepath.Join(dir, "podman"), []byte("#!/bin/sh\n: > "+marker+"\nexit 0\n"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir)
+	r := (&Executor{Env: AllowlistEnv{}}).Run(context.Background(), JobSpec{AttemptID: "podman", CoordinatorID: "coord", Home: t.TempDir(),
+		Job: model.Job{Isolation: "container", Container: &model.ContainerConfig{Image: "alpine:3.19"}}, Command: []string{"true"}})
+	if r.Err != nil {
+		t.Fatal(r.Err)
+	}
+	if _, err := os.Stat(marker); err != nil {
+		t.Fatalf("podman was not run: %v", err)
+	}
+}

free/ci/internal/toolchain/image.go

@@ -31,17 +31,23 @@ var refRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._/:-]*@sha256:[0-9a-f]{64}$`)
 // only when the process could not run (the doctor and report funnel).
 type Capture func(ctx context.Context, argv ...string) (string, int, error)
 
+// rcNotFound is the exit code of the shell when the command is not installed.
+const rcNotFound = 127
+
 // Resolver turns the environment and the pins into an image reference.
 type Resolver struct {
 	Getenv   func(string) string
 	ReadFile func(string) ([]byte, error)
 	Capture  Capture // nil: do not check or pull the image
+	// Runtimes are the container CLIs tried in order; the first one that can run
+	// is used for the inspect and the pull. Empty means docker, then podman.
+	Runtimes []string
 
 	mu   sync.Mutex
 	have map[string]bool
 }
 
-// NewResolver reads the real process environment and uses capture for docker.
+// NewResolver reads the real process environment and uses capture for docker or podman.
 func NewResolver(capture Capture) *Resolver {
 	return &Resolver{Getenv: os.Getenv, ReadFile: os.ReadFile, Capture: capture}
 }
@@ -95,7 +101,7 @@ func (r *Resolver) table() (Versions, error) {
 	return ParseVersions(b)
 }
 
-// ensure makes ref present on the host. docker verifies the manifest digest
+// ensure makes ref present on the host. The container runtime verifies the manifest digest
 // while pulling, so content that does not match the pin never lands.
 func (r *Resolver) ensure(ctx context.Context, ref string) error {
 	if r.Capture == nil {
@@ -106,10 +112,25 @@ func (r *Resolver) ensure(ctx context.Context, ref string) error {
 	if r.have[ref] {
 		return nil
 	}
-	if _, rc, err := r.Capture(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", ref); err == nil && rc == 0 {
-		return r.remember(ref)
+	runtimes := r.Runtimes
+	if len(runtimes) == 0 {
+		runtimes = []string{"docker", "podman"}
+	}
+	for i, rt := range runtimes {
+		_, rc, err := r.Capture(ctx, rt, "image", "inspect", "--format", "{{.Id}}", ref)
+		if (err != nil || rc == rcNotFound) && i < len(runtimes)-1 {
+			continue // this CLI is not installed: try the next one
+		}
+		if err == nil && rc == 0 {
+			return r.remember(ref)
+		}
+		return r.pull(ctx, rt, ref)
 	}
-	out, rc, err := r.Capture(ctx, "docker", "pull", "--quiet", ref)
+	return nil // unreachable: the last runtime always returns above
+}
+
+func (r *Resolver) pull(ctx context.Context, runtime, ref string) error {
+	out, rc, err := r.Capture(ctx, runtime, "pull", "--quiet", ref)
 	if err != nil || rc != 0 {
 		return fail("cannot pull %s (digest mismatch, unknown digest or registry unreachable): %s", ref, tail(out, err))
 	}

free/ci/internal/toolchain/image_test.go

@@ -105,3 +105,38 @@ func TestEnsurePullsOnceAndMapsFailureToE645(t *testing.T) {
 		t.Fatalf("failed pull error = %v", err)
 	}
 }
+
+// D-1185: with no docker CLI the image is inspected and pulled with podman.
+func TestEnsureFallsBackToPodmanWhenDockerCannotRun(t *testing.T) {
+	ref := "docker.io/library/alpine:3.22@" + pinnedDigest
+	env := envOf(map[string]string{EnvImage: ref})
+	var calls []string
+	capture := func(_ context.Context, argv ...string) (string, int, error) {
+		calls = append(calls, argv[0]+" "+argv[1])
+		if argv[0] == "docker" { // the exec funnel reports a missing CLI as the shell's 127
+			return "ci-job: line 0: exec: docker: not found", 127, nil
+		}
+		if argv[1] == "image" {
+			return "", 1, nil
+		}
+		return "pulled", 0, nil
+	}
+	if got, err := (&Resolver{Getenv: env, Capture: capture}).Resolve(context.Background()); err != nil || got != ref {
+		t.Fatalf("resolve: %q %v", got, err)
+	}
+	if want := "docker image,podman image,podman pull"; strings.Join(calls, ",") != want {
+		t.Fatalf("calls %v, want %s", calls, want)
+	}
+
+	calls = nil
+	present := func(_ context.Context, argv ...string) (string, int, error) {
+		calls = append(calls, argv[0]+" "+argv[1])
+		if argv[0] == "docker" {
+			return "", -1, errors.New("executable file not found")
+		}
+		return "", 0, nil
+	}
+	if _, err := (&Resolver{Getenv: env, Capture: present}).Resolve(context.Background()); err != nil || len(calls) != 2 {
+		t.Fatalf("image present under podman: %v calls %v", err, calls)
+	}
+}