From 467529fae2639c471ae67fdf441bf7f4064ca530 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 11:24:53 +0000
Subject: [PATCH 1/6] Add service container commands

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 agent/internal/agent/handlers.go              |  17 +
 agent/internal/agent/workqueue.go             |  32 +-
 agent/internal/container/runtime.go           |  84 +++++
 agent/internal/container/runtime_test.go      |  44 +++
 agent/internal/http/client.go                 |  12 +-
 .../services/[serviceId]/commands/page.tsx    | 310 ++++++++++++++++++
 web/app/api/inngest/route.ts                  |   2 +
 web/app/api/projects/[id]/services/route.ts   |   6 +-
 web/app/api/services/[id]/commands/route.ts   | 177 ++++++++++
 .../service/service-layout-client.tsx         |   4 +-
 web/db/schema.ts                              |  39 +++
 web/db/types.ts                               |   4 +-
 web/lib/inngest/functions/crons.ts            |  11 +
 web/lib/inngest/functions/index.ts            |   1 +
 web/lib/navigation.ts                         |   6 +
 web/lib/scheduler.ts                          |  44 ++-
 web/lib/service-command-retention.ts          |  18 +
 web/lib/work-queue.ts                         |  55 +++-
 web/tests/inngest-route.test.ts               |   1 +
 web/tests/navigation.test.ts                  |   3 +-
 web/tests/service-command-retention.test.ts   |  52 +++
 web/tests/service-commands-route.test.ts      | 181 ++++++++++
 web/tests/work-queue.test.ts                  |  95 +++++-
 23 files changed, 1174 insertions(+), 24 deletions(-)
 create mode 100644 web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
 create mode 100644 web/app/api/services/[id]/commands/route.ts
 create mode 100644 web/lib/service-command-retention.ts
 create mode 100644 web/tests/service-command-retention.test.ts
 create mode 100644 web/tests/service-commands-route.test.ts

diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go
index 0b8aeb29..3686144d 100644
--- a/agent/internal/agent/handlers.go
+++ b/agent/internal/agent/handlers.go
@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"time"
+	"unicode/utf8"
 
 	"techulus/cloud-agent/internal/build"
 	"techulus/cloud-agent/internal/container"
@@ -19,6 +20,22 @@ import (
 	"techulus/cloud-agent/internal/registryauth"
 )
 
+func (a *Agent) ProcessCommand(item agenthttp.WorkQueueItem) (container.CommandResult, error) {
+	var payload struct {
+		CommandRunID string `json:"commandRunId"`
+		DeploymentID string `json:"deploymentId"`
+		ContainerID  string `json:"containerId"`
+		Command      string `json:"command"`
+	}
+	if err := json.Unmarshal([]byte(item.Payload), &payload); err != nil {
+		return container.CommandResult{}, fmt.Errorf("failed to parse command payload: %w", err)
+	}
+	if payload.CommandRunID != item.ID || payload.DeploymentID == "" || payload.ContainerID == "" || payload.Command == "" || utf8.RuneCountInString(payload.Command) > 4096 {
+		return container.CommandResult{}, fmt.Errorf("invalid command payload")
+	}
+	return container.ExecCommand(payload.ContainerID, payload.Command)
+}
+
 func (a *Agent) ProcessRestart(item agenthttp.WorkQueueItem) error {
 	var payload struct {
 		DeploymentID string `json:"deploymentId"`
diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go
index 5791a21f..2f6411c5 100644
--- a/agent/internal/agent/workqueue.go
+++ b/agent/internal/agent/workqueue.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"time"
 
+	"techulus/cloud-agent/internal/container"
 	agenthttp "techulus/cloud-agent/internal/http"
 )
 
@@ -93,7 +94,19 @@ func (a *Agent) processLeasedWorkItem(item agenthttp.WorkQueueItem) {
 	status := "completed"
 	errorMsg := ""
 	restartAfterReport := false
-	if err := a.ProcessWorkItem(item); err != nil {
+	var commandResult *container.CommandResult
+	var processErr error
+	if item.Type == "command" {
+		result, err := a.ProcessCommand(item)
+		processErr = err
+		if err == nil {
+			commandResult = &result
+		}
+	} else {
+		processErr = a.ProcessWorkItem(item)
+	}
+	if processErr != nil {
+		err := processErr
 		if errors.Is(err, errAgentUpgradeRestartNeeded) {
 			restartAfterReport = true
 		} else {
@@ -109,12 +122,25 @@ func (a *Agent) processLeasedWorkItem(item agenthttp.WorkQueueItem) {
 	if !restartAfterReport && a.activeWorkItem != nil && a.activeWorkItem.ID == item.ID && a.activeWorkItem.Attempt == item.Attempt {
 		a.activeWorkItem = nil
 	}
-	a.pendingWorkResults = append(a.pendingWorkResults, agenthttp.CompletedWorkItem{
+	completed := agenthttp.CompletedWorkItem{
 		ID:      item.ID,
 		Attempt: item.Attempt,
 		Status:  status,
 		Error:   errorMsg,
-	})
+	}
+	if commandResult != nil {
+		completed.Output = commandResult.Output
+		completed.ExitCode = &commandResult.ExitCode
+		completed.OutputTruncated = commandResult.Truncated
+		if commandResult.TimedOut {
+			completed.Status = "failed"
+			completed.Error = "command timed out after 60 seconds"
+			completed.TimedOut = true
+		} else if commandResult.ExitCode != 0 {
+			completed.Status = "failed"
+		}
+	}
+	a.pendingWorkResults = append(a.pendingWorkResults, completed)
 	a.workMutex.Unlock()
 
 	a.RequestStatusReport("work item " + status)
diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go
index ee4d08fb..e1212fb0 100644
--- a/agent/internal/container/runtime.go
+++ b/agent/internal/container/runtime.go
@@ -1,6 +1,7 @@
 package container
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -9,12 +10,95 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"sync"
 	"time"
+	"unicode/utf8"
 
 	"techulus/cloud-agent/internal/retry"
 	"techulus/cloud-agent/internal/wireguard"
 )
 
+const CommandOutputLimit = 64 * 1024
+
+var commandTimeout = 60 * time.Second
+
+type CommandResult struct {
+	Output    string
+	ExitCode  int
+	Truncated bool
+	TimedOut  bool
+}
+
+type limitedBuffer struct {
+	mutex     sync.Mutex
+	buffer    bytes.Buffer
+	truncated bool
+}
+
+func (b *limitedBuffer) snapshot() (string, bool) {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	return b.buffer.String(), b.truncated
+}
+
+func (b *limitedBuffer) Write(p []byte) (int, error) {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+
+	n := len(p)
+	remaining := CommandOutputLimit - b.buffer.Len()
+	if remaining > 0 {
+		writeLength := min(remaining, len(p))
+		_, _ = b.buffer.Write(p[:writeLength])
+		if writeLength < len(p) {
+			b.truncated = true
+		}
+	} else if len(p) > 0 {
+		b.truncated = true
+	}
+	return n, nil
+}
+
+func ExecCommand(containerID, command string) (CommandResult, error) {
+	running, err := IsContainerRunning(containerID)
+	if err != nil {
+		return CommandResult{}, err
+	}
+	if !running {
+		return CommandResult{}, fmt.Errorf("container is not running")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "podman", "exec", containerID, "/bin/sh", "-c", command)
+	var output limitedBuffer
+	cmd.Stdout, cmd.Stderr = &output, &output
+	err = cmd.Run()
+	outputText, truncated := output.snapshot()
+	outputText = strings.ToValidUTF8(outputText, "�")
+	if len(outputText) > CommandOutputLimit {
+		outputText = outputText[:CommandOutputLimit]
+		for !utf8.ValidString(outputText) {
+			outputText = outputText[:len(outputText)-1]
+		}
+		truncated = true
+	}
+	result := CommandResult{Output: outputText, Truncated: truncated}
+	if ctx.Err() == context.DeadlineExceeded {
+		result.TimedOut = true
+		result.ExitCode = 124
+		return result, nil
+	}
+	if err == nil {
+		return result, nil
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		result.ExitCode = exitErr.ExitCode()
+		return result, nil
+	}
+	return result, fmt.Errorf("failed to execute command: %w", err)
+}
+
 func ContainerExists(containerID string) (bool, error) {
 	cmd := exec.Command("podman", "inspect", "--format", "json", containerID)
 	output, err := cmd.CombinedOutput()
diff --git a/agent/internal/container/runtime_test.go b/agent/internal/container/runtime_test.go
index f983aba5..7055c154 100644
--- a/agent/internal/container/runtime_test.go
+++ b/agent/internal/container/runtime_test.go
@@ -8,8 +8,52 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
+func TestExecCommand(t *testing.T) {
+	dir := t.TempDir()
+	script := `#!/bin/sh
+if [ "$1" = inspect ]; then echo '[{"State":{"Running":true}}]'; exit 0; fi
+case "$5" in
+ success) printf 'hello';;
+ failure) printf 'bad'; exit 7;;
+ exact) head -c 65536 /dev/zero | tr '\0' x;;
+ large) head -c 70000 /dev/zero | tr '\0' x;;
+ timeout) sleep 1;;
+esac`
+	if err := os.WriteFile(filepath.Join(dir, "podman"), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
+	tests := []struct {
+		name                string
+		exit                int
+		truncated, timedOut bool
+	}{
+		{"success", 0, false, false}, {"failure", 7, false, false}, {"exact", 0, false, false}, {"large", 0, true, false}, {"timeout", 124, false, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			old := commandTimeout
+			if tt.timedOut {
+				commandTimeout = 10 * time.Millisecond
+			}
+			defer func() { commandTimeout = old }()
+			result, err := ExecCommand("container", tt.name)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if result.ExitCode != tt.exit || result.Truncated != tt.truncated || result.TimedOut != tt.timedOut {
+				t.Fatalf("unexpected result: %+v", result)
+			}
+			if len(result.Output) > CommandOutputLimit {
+				t.Fatalf("output exceeded limit: %d", len(result.Output))
+			}
+		})
+	}
+}
+
 func TestBuildPodmanPullArgs(t *testing.T) {
 	tests := []struct {
 		name      string
diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go
index 5cf21ede..b825e720 100644
--- a/agent/internal/http/client.go
+++ b/agent/internal/http/client.go
@@ -358,10 +358,14 @@ type StatusReport struct {
 }
 
 type CompletedWorkItem struct {
-	ID      string `json:"id"`
-	Attempt int    `json:"attempt"`
-	Status  string `json:"status"`
-	Error   string `json:"error,omitempty"`
+	ID              string `json:"id"`
+	Attempt         int    `json:"attempt"`
+	Status          string `json:"status"`
+	Error           string `json:"error,omitempty"`
+	Output          string `json:"output,omitempty"`
+	ExitCode        *int   `json:"exitCode,omitempty"`
+	OutputTruncated bool   `json:"outputTruncated,omitempty"`
+	TimedOut        bool   `json:"timedOut,omitempty"`
 }
 
 type ActiveWorkItem struct {
diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
new file mode 100644
index 00000000..52e2a7aa
--- /dev/null
+++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
@@ -0,0 +1,310 @@
+"use client";
+
+import { Loader2, Terminal, XCircle } from "lucide-react";
+import { useMemo, useState } from "react";
+import useSWRInfinite from "swr/infinite";
+import { useService } from "@/components/service/service-layout-client";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import {
+	Empty,
+	EmptyDescription,
+	EmptyMedia,
+	EmptyTitle,
+} from "@/components/ui/empty";
+import {
+	NativeSelect,
+	NativeSelectOption,
+} from "@/components/ui/native-select";
+import { Textarea } from "@/components/ui/textarea";
+import { formatDateTime, formatRelativeTime } from "@/lib/date";
+import { isObservedReady } from "@/lib/deployment-status";
+import { fetcher } from "@/lib/fetcher";
+
+type CommandStatus =
+	| "pending"
+	| "running"
+	| "succeeded"
+	| "failed"
+	| "timed_out";
+
+type CommandRun = {
+	id: string;
+	command: string;
+	status: CommandStatus;
+	output: string | null;
+	exitCode: number | null;
+	outputTruncated: boolean;
+	errorMessage: string | null;
+	actor: { name: string };
+	serverName: string;
+	containerId: string;
+	createdAt: string;
+	startedAt: string | null;
+	completedAt: string | null;
+};
+
+type CommandHistory = {
+	commands: CommandRun[];
+	nextCursor: string | null;
+};
+
+const STATUS_LABELS: Record<CommandStatus, string> = {
+	pending: "Queued",
+	running: "Running",
+	succeeded: "Succeeded",
+	failed: "Failed",
+	timed_out: "Timed out",
+};
+
+function CommandStatusBadge({ command }: { command: CommandRun }) {
+	const failed = command.status === "failed" || command.status === "timed_out";
+	return (
+		<Badge
+			variant={failed ? "destructive" : "secondary"}
+			className={
+				command.status === "succeeded" ? "text-emerald-700" : undefined
+			}
+		>
+			{STATUS_LABELS[command.status]}
+			{command.exitCode !== null ? ` · exit ${command.exitCode}` : null}
+		</Badge>
+	);
+}
+
+export default function CommandsPage() {
+	const { service } = useService();
+	const targets = service.deployments.filter(
+		(deployment) =>
+			deployment.containerId &&
+			deployment.runtimeDesiredState === "running" &&
+			isObservedReady(deployment.observedPhase) &&
+			deployment.server?.status === "online",
+	);
+	const [deploymentId, setDeploymentId] = useState(targets[0]?.id ?? "");
+	const selectedDeploymentId = targets.some(
+		(target) => target.id === deploymentId,
+	)
+		? deploymentId
+		: (targets[0]?.id ?? "");
+	const [command, setCommand] = useState("");
+	const [submitting, setSubmitting] = useState(false);
+	const [submitError, setSubmitError] = useState<string | null>(null);
+
+	const {
+		data,
+		error: historyError,
+		isLoading,
+		isValidating,
+		mutate,
+		size,
+		setSize,
+	} = useSWRInfinite<CommandHistory>(
+		(pageIndex, previousPage) => {
+			if (previousPage && !previousPage.nextCursor) return null;
+			const cursor = pageIndex === 0 ? null : previousPage?.nextCursor;
+			return `/api/services/${service.id}/commands${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ""}`;
+		},
+		fetcher,
+		{
+			refreshInterval: (pages) =>
+				pages?.some((page) =>
+					page.commands.some(
+						(item) => item.status === "pending" || item.status === "running",
+					),
+				)
+					? 2000
+					: 0,
+			revalidateOnFocus: true,
+		},
+	);
+
+	const history = useMemo(() => {
+		const byId = new Map<string, CommandRun>();
+		for (const page of data ?? []) {
+			for (const item of page.commands) byId.set(item.id, item);
+		}
+		return [...byId.values()];
+	}, [data]);
+	const hasMore = data?.[data.length - 1]?.nextCursor != null;
+	const isLoadingMore = isValidating && Boolean(data?.[size - 1] === undefined);
+
+	async function runCommand() {
+		setSubmitting(true);
+		setSubmitError(null);
+		try {
+			const response = await fetch(`/api/services/${service.id}/commands`, {
+				method: "POST",
+				headers: { "content-type": "application/json" },
+				body: JSON.stringify({
+					deploymentId: selectedDeploymentId,
+					command,
+				}),
+			});
+			if (!response.ok) {
+				const body = (await response.json()) as {
+					error?: string;
+					message?: string;
+				};
+				throw new Error(
+					body.error ?? body.message ?? "Command could not be queued",
+				);
+			}
+			setCommand("");
+			await mutate();
+		} catch (error) {
+			setSubmitError(
+				error instanceof Error ? error.message : "Command could not be queued",
+			);
+		} finally {
+			setSubmitting(false);
+		}
+	}
+
+	return (
+		<div className="space-y-6">
+			<Card>
+				<CardHeader>
+					<CardTitle>Run command</CardTitle>
+				</CardHeader>
+				<CardContent className="space-y-4">
+					{targets.length === 0 ? (
+						<p className="text-sm text-muted-foreground">
+							No ready, running container is available on an online server.
+						</p>
+					) : (
+						<NativeSelect
+							aria-label="Target container"
+							value={selectedDeploymentId}
+							onChange={(event) => setDeploymentId(event.target.value)}
+						>
+							{targets.map((target) => (
+								<NativeSelectOption key={target.id} value={target.id}>
+									{target.server?.name} · {target.containerId?.slice(0, 12)}
+								</NativeSelectOption>
+							))}
+						</NativeSelect>
+					)}
+					<Textarea
+						aria-label="Command"
+						value={command}
+						onChange={(event) => setCommand(event.target.value)}
+						maxLength={4096}
+						rows={4}
+						placeholder="e.g. ls -la"
+						disabled={targets.length === 0 || submitting}
+					/>
+					<div className="flex items-center justify-between gap-4">
+						<span className="text-xs text-muted-foreground">
+							{command.length}/4096 · 60 second timeout
+						</span>
+						<Button
+							onClick={() => void runCommand()}
+							disabled={!selectedDeploymentId || !command.trim() || submitting}
+						>
+							{submitting ? <Loader2 className="animate-spin" /> : null}
+							{submitting ? "Queuing…" : "Run"}
+						</Button>
+					</div>
+					{submitError ? (
+						<p className="text-sm text-destructive">{submitError}</p>
+					) : null}
+				</CardContent>
+			</Card>
+
+			<section className="space-y-3">
+				<h2 className="text-lg font-semibold">Command history</h2>
+				{isLoading ? (
+					<p className="text-sm text-muted-foreground">Loading history…</p>
+				) : historyError ? (
+					<Empty className="border py-12">
+						<EmptyMedia variant="icon">
+							<XCircle />
+						</EmptyMedia>
+						<EmptyTitle>Unable to load command history</EmptyTitle>
+						<EmptyDescription>
+							Command history is unavailable or you do not have permission to
+							view it.
+						</EmptyDescription>
+						<Button variant="outline" onClick={() => void mutate()}>
+							Retry
+						</Button>
+					</Empty>
+				) : history.length === 0 ? (
+					<Empty className="border py-12">
+						<EmptyMedia variant="icon">
+							<Terminal />
+						</EmptyMedia>
+						<EmptyTitle>No commands yet</EmptyTitle>
+						<EmptyDescription>
+							Run a command to see its output and execution details here.
+						</EmptyDescription>
+					</Empty>
+				) : (
+					<div className="space-y-3">
+						{history.map((item) => (
+							<Card key={item.id}>
+								<CardContent className="space-y-3 pt-6">
+									<div className="flex items-start justify-between gap-4">
+										<code className="break-all text-sm">$ {item.command}</code>
+										<CommandStatusBadge command={item} />
+									</div>
+									<p className="text-xs text-muted-foreground">
+										{item.actor.name} · {item.serverName} ·{" "}
+										{item.containerId.slice(0, 12)} ·{" "}
+										<time title={formatDateTime(item.createdAt)}>
+											{formatRelativeTime(item.createdAt)}
+										</time>
+									</p>
+									{item.startedAt || item.completedAt ? (
+										<p className="text-xs text-muted-foreground">
+											{item.startedAt
+												? `Started ${formatDateTime(item.startedAt)}`
+												: null}
+											{item.startedAt && item.completedAt ? " · " : null}
+											{item.completedAt
+												? `Completed ${formatDateTime(item.completedAt)}`
+												: null}
+										</p>
+									) : null}
+									{item.output || item.errorMessage ? (
+										<details open>
+											<summary className="cursor-pointer text-sm">
+												Combined output
+											</summary>
+											<pre className="mt-2 max-h-96 overflow-auto whitespace-pre-wrap rounded-md bg-muted p-3 text-xs">
+												{item.output}
+												{item.errorMessage
+													? `${item.output ? "\n" : ""}${item.errorMessage}`
+													: null}
+											</pre>
+											{item.outputTruncated ? (
+												<p className="mt-1 text-xs text-amber-600">
+													Output was truncated at 64 KiB.
+												</p>
+											) : null}
+										</details>
+									) : null}
+								</CardContent>
+							</Card>
+						))}
+					</div>
+				)}
+
+				{hasMore ? (
+					<div className="flex justify-center pt-2">
+						<Button
+							variant="outline"
+							disabled={isLoadingMore}
+							onClick={() => void setSize(size + 1)}
+						>
+							{isLoadingMore ? <Loader2 className="animate-spin" /> : null}
+							Load older commands
+						</Button>
+					</div>
+				) : null}
+			</section>
+		</div>
+	);
+}
diff --git a/web/app/api/inngest/route.ts b/web/app/api/inngest/route.ts
index e1e47380..99212b54 100644
--- a/web/app/api/inngest/route.ts
+++ b/web/app/api/inngest/route.ts
@@ -23,6 +23,7 @@ import {
 	scheduledBackupsCheck,
 	scheduledDeploymentsCheck,
 	serviceDeletionWorkflow,
+	serviceCommandRetention,
 	serviceRestoreWorkflow,
 	staleItemsCleanup,
 	staleServerCheck,
@@ -35,6 +36,7 @@ export const { GET, POST, PUT } = serve({
 		rolloutWorkflow,
 		onDeploymentFailed,
 		staleServerCheck,
+		serviceCommandRetention,
 		scheduledDeploymentsCheck,
 		certificateRenewal,
 		challengeCleanup,
diff --git a/web/app/api/projects/[id]/services/route.ts b/web/app/api/projects/[id]/services/route.ts
index a9941f38..cb44107a 100644
--- a/web/app/api/projects/[id]/services/route.ts
+++ b/web/app/api/projects/[id]/services/route.ts
@@ -329,7 +329,11 @@ export async function GET(
 							.from(deploymentPorts)
 							.where(eq(deploymentPorts.deploymentId, deployment.id)),
 						db
-							.select({ name: servers.name, wireguardIp: servers.wireguardIp })
+							.select({
+								name: servers.name,
+								wireguardIp: servers.wireguardIp,
+								status: servers.status,
+							})
 							.from(servers)
 							.where(eq(servers.id, deployment.serverId))
 							.then((r) => r[0]),
diff --git a/web/app/api/services/[id]/commands/route.ts b/web/app/api/services/[id]/commands/route.ts
new file mode 100644
index 00000000..dca16557
--- /dev/null
+++ b/web/app/api/services/[id]/commands/route.ts
@@ -0,0 +1,177 @@
+import { randomUUID } from "node:crypto";
+import { and, desc, eq, inArray, isNull, lt, or, sql } from "drizzle-orm";
+import { db } from "@/db";
+import { deployments, servers, serviceCommands, services } from "@/db/schema";
+import { requireRequestDeveloperRole } from "@/lib/api-auth";
+import { observedReadyPhases } from "@/lib/deployment-status";
+import {
+	decodeTimestampCursor,
+	encodeTimestampCursor,
+} from "@/lib/public-api-pagination";
+import { enqueueWork } from "@/lib/work-queue";
+
+const PAGE_SIZE = 25;
+const MAX_COMMAND_LENGTH = 4096;
+
+export async function GET(
+	request: Request,
+	{ params }: { params: Promise<{ id: string }> },
+) {
+	const auth = await requireRequestDeveloperRole(request);
+	if (!auth.ok) return auth.response;
+
+	const { id: serviceId } = await params;
+	const cursorValue = new URL(request.url).searchParams.get("cursor");
+	const cursor = decodeTimestampCursor(cursorValue);
+	if (cursorValue && !cursor) {
+		return Response.json({ error: "Invalid cursor" }, { status: 400 });
+	}
+
+	const rows = await db
+		.select({
+			id: serviceCommands.id,
+			command: serviceCommands.command,
+			status: serviceCommands.status,
+			output: serviceCommands.output,
+			exitCode: serviceCommands.exitCode,
+			outputTruncated: serviceCommands.outputTruncated,
+			errorMessage: serviceCommands.errorMessage,
+			actor: serviceCommands.actor,
+			serverName: serviceCommands.serverName,
+			containerId: serviceCommands.containerId,
+			createdAt: serviceCommands.createdAt,
+			cursorCreatedAt: sql<string>`${serviceCommands.createdAt}::text`,
+			startedAt: serviceCommands.startedAt,
+			completedAt: serviceCommands.completedAt,
+		})
+		.from(serviceCommands)
+		.where(
+			and(
+				eq(serviceCommands.serviceId, serviceId),
+				cursor
+					? or(
+							lt(
+								serviceCommands.createdAt,
+								sql`${cursor.createdAt}::timestamptz`,
+							),
+							and(
+								eq(
+									serviceCommands.createdAt,
+									sql`${cursor.createdAt}::timestamptz`,
+								),
+								lt(serviceCommands.id, cursor.id),
+							),
+						)
+					: undefined,
+			),
+		)
+		.orderBy(desc(serviceCommands.createdAt), desc(serviceCommands.id))
+		.limit(PAGE_SIZE + 1);
+
+	const pageRows = rows.slice(0, PAGE_SIZE);
+	const last = pageRows.at(-1);
+	return Response.json({
+		commands: pageRows.map(({ cursorCreatedAt: _, actor, ...command }) => ({
+			...command,
+			actor: { name: actor.name },
+		})),
+		nextCursor:
+			rows.length > PAGE_SIZE && last
+				? encodeTimestampCursor({
+						createdAt: last.cursorCreatedAt,
+						id: last.id,
+					})
+				: null,
+	});
+}
+
+export async function POST(
+	request: Request,
+	{ params }: { params: Promise<{ id: string }> },
+) {
+	const auth = await requireRequestDeveloperRole(request);
+	if (!auth.ok) return auth.response;
+
+	const { id: serviceId } = await params;
+	let body: { deploymentId?: unknown; command?: unknown };
+	try {
+		body = await request.json();
+	} catch {
+		return Response.json({ error: "Invalid JSON" }, { status: 400 });
+	}
+
+	if (
+		typeof body.command !== "string" ||
+		body.command.trim().length === 0 ||
+		body.command.length > MAX_COMMAND_LENGTH ||
+		typeof body.deploymentId !== "string"
+	) {
+		return Response.json(
+			{ error: "A deployment and command of 1-4096 characters are required" },
+			{ status: 400 },
+		);
+	}
+
+	const service = await db
+		.select({ id: services.id })
+		.from(services)
+		.where(and(eq(services.id, serviceId), isNull(services.deletedAt)))
+		.then((rows) => rows[0]);
+	if (!service) {
+		return Response.json({ error: "Service not found" }, { status: 404 });
+	}
+
+	const target = await db
+		.select({
+			deploymentId: deployments.id,
+			serverId: deployments.serverId,
+			containerId: deployments.containerId,
+			serverName: servers.name,
+		})
+		.from(deployments)
+		.innerJoin(servers, eq(servers.id, deployments.serverId))
+		.where(
+			and(
+				eq(deployments.id, body.deploymentId),
+				eq(deployments.serviceId, serviceId),
+				eq(deployments.runtimeDesiredState, "running"),
+				inArray(deployments.observedPhase, observedReadyPhases),
+				eq(servers.status, "online"),
+			),
+		)
+		.then((rows) => rows[0]);
+	if (!target?.containerId) {
+		return Response.json(
+			{ error: "Container is no longer runnable" },
+			{ status: 409 },
+		);
+	}
+	const containerId = target.containerId;
+
+	const commandId = randomUUID();
+	await db.transaction(async (tx) => {
+		await tx.insert(serviceCommands).values({
+			id: commandId,
+			serviceId,
+			deploymentId: target.deploymentId,
+			serverId: target.serverId,
+			serverName: target.serverName,
+			containerId,
+			actor: { id: auth.session.user.id, name: auth.session.user.name },
+			command: body.command as string,
+		});
+		await enqueueWork(
+			target.serverId,
+			"command",
+			{
+				commandRunId: commandId,
+				deploymentId: target.deploymentId,
+				containerId,
+				command: body.command as string,
+			},
+			{ id: commandId, tx },
+		);
+	});
+
+	return Response.json({ id: commandId, status: "pending" }, { status: 202 });
+}
diff --git a/web/components/service/service-layout-client.tsx b/web/components/service/service-layout-client.tsx
index c0237fd4..563d82ed 100644
--- a/web/components/service/service-layout-client.tsx
+++ b/web/components/service/service-layout-client.tsx
@@ -144,7 +144,8 @@ export function ServiceLayoutClient({
 		pathname.includes("/configuration") ||
 		pathname.includes("/changelog") ||
 		pathname.includes("/builds") ||
-		pathname.includes("/backups");
+		pathname.includes("/backups") ||
+		pathname.includes("/commands");
 
 	const hasPublicPorts = service?.ports?.some((p) => p.isPublic);
 
@@ -153,6 +154,7 @@ export function ServiceLayoutClient({
 		{ name: "Configuration", href: `${basePath}/configuration` },
 		{ name: "Metrics", href: `${basePath}/metrics` },
 		{ name: "Logs", href: `${basePath}/logs` },
+		{ name: "Commands", href: `${basePath}/commands` },
 		...(hasPublicPorts
 			? [{ name: "Requests", href: `${basePath}/requests` }]
 			: []),
diff --git a/web/db/schema.ts b/web/db/schema.ts
index 221eaeaa..82f23b87 100644
--- a/web/db/schema.ts
+++ b/web/db/schema.ts
@@ -910,6 +910,7 @@ export const workQueue = pgTable(
 				"create_manifest",
 				"upgrade_agent",
 				"sync_registries",
+				"command",
 			],
 		}).notNull(),
 		payload: text("payload").notNull(),
@@ -943,6 +944,44 @@ export const workQueue = pgTable(
 	],
 );
 
+export const serviceCommands = pgTable(
+	"service_commands",
+	{
+		id: text("id").primaryKey(),
+		serviceId: text("service_id")
+			.notNull()
+			.references(() => services.id, { onDelete: "cascade" }),
+		deploymentId: text("deployment_id").notNull(),
+		serverId: text("server_id").notNull(),
+		serverName: text("server_name").notNull(),
+		containerId: text("container_id").notNull(),
+		actor: jsonb("actor").$type<{ id: string; name: string }>().notNull(),
+		command: text("command").notNull(),
+		status: text("status", {
+			enum: ["pending", "running", "succeeded", "failed", "timed_out"],
+		})
+			.notNull()
+			.default("pending"),
+		output: text("output"),
+		exitCode: integer("exit_code"),
+		outputTruncated: boolean("output_truncated").notNull().default(false),
+		errorMessage: text("error_message"),
+		createdAt: timestamp("created_at", { withTimezone: true })
+			.defaultNow()
+			.notNull(),
+		startedAt: timestamp("started_at", { withTimezone: true }),
+		completedAt: timestamp("completed_at", { withTimezone: true }),
+	},
+	(table) => [
+		index("service_commands_service_created_id_idx").on(
+			table.serviceId,
+			table.createdAt,
+			table.id,
+		),
+		index("service_commands_created_at_idx").on(table.createdAt),
+	],
+);
+
 export const githubInstallations = pgTable(
 	"github_installations",
 	{
diff --git a/web/db/types.ts b/web/db/types.ts
index 49a9811b..875fedf4 100644
--- a/web/db/types.ts
+++ b/web/db/types.ts
@@ -12,6 +12,7 @@ import type {
 	servers,
 	servicePorts,
 	serviceReplicas,
+	serviceCommands,
 	services,
 	serviceVolumes,
 	user,
@@ -25,6 +26,7 @@ export type Service = typeof services.$inferSelect;
 export type ServicePort = typeof servicePorts.$inferSelect;
 export type ServiceVolume = typeof serviceVolumes.$inferSelect;
 export type ServiceReplica = typeof serviceReplicas.$inferSelect;
+export type ServiceCommand = typeof serviceCommands.$inferSelect;
 export type Secret = typeof secrets.$inferSelect;
 export type Deployment = typeof deployments.$inferSelect;
 export type DeploymentPort = typeof deploymentPorts.$inferSelect;
@@ -50,7 +52,7 @@ export type ServiceWithDetails = Service & {
 	>;
 	deployments: Array<
 		Deployment & {
-			server: Pick<Server, "name" | "wireguardIp"> | null;
+			server: Pick<Server, "name" | "wireguardIp" | "status"> | null;
 			ports: Array<
 				Pick<DeploymentPort, "id" | "hostPort"> & { containerPort: number }
 			>;
diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts
index 16079d9e..ddcd433b 100644
--- a/web/lib/inngest/functions/crons.ts
+++ b/web/lib/inngest/functions/crons.ts
@@ -7,6 +7,7 @@ import { cleanupOldBackups, runScheduledBackups } from "@/lib/backup-scheduler";
 import { checkAndPersistControlPlaneUpdate } from "@/lib/control-plane-updates";
 import { cleanupReadNotifications } from "@/lib/notifications";
 import { cleanupRegistryArtifactsDaily } from "@/lib/registry-retention";
+import { cleanupOldServiceCommands } from "@/lib/service-command-retention";
 import {
 	checkAndRecoverStaleServers,
 	checkAndRunScheduledDeployments,
@@ -195,3 +196,13 @@ export const notificationRetention = inngest.createFunction(
 		await step.run("cleanup-read-notifications", cleanupReadNotifications);
 	},
 );
+
+export const serviceCommandRetention = inngest.createFunction(
+	{
+		id: "cron-service-command-retention",
+		triggers: [cron("0 7 * * *")],
+		singleton: { mode: "skip" },
+	},
+	async ({ step }) =>
+		step.run("cleanup-service-commands", cleanupOldServiceCommands),
+);
diff --git a/web/lib/inngest/functions/index.ts b/web/lib/inngest/functions/index.ts
index 04bbea4d..51caadfa 100644
--- a/web/lib/inngest/functions/index.ts
+++ b/web/lib/inngest/functions/index.ts
@@ -8,6 +8,7 @@ export {
 	challengeCleanup,
 	controlPlaneUpdateCheck,
 	notificationRetention,
+	serviceCommandRetention,
 	oldBackupsCleanup,
 	registryArtifactRetention,
 	scheduledBackupsCheck,
diff --git a/web/lib/navigation.ts b/web/lib/navigation.ts
index 486aba12..bcaf9009 100644
--- a/web/lib/navigation.ts
+++ b/web/lib/navigation.ts
@@ -95,6 +95,12 @@ const servicePages = [
 	},
 	{ key: "metrics", label: "Metrics", suffix: "/metrics", aliases: ["usage"] },
 	{ key: "logs", label: "Logs", suffix: "/logs", aliases: ["output"] },
+	{
+		key: "commands",
+		label: "Commands",
+		suffix: "/commands",
+		aliases: ["shell", "exec"],
+	},
 	{
 		key: "requests",
 		label: "Requests",
diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts
index 6593e6b9..ab478e25 100644
--- a/web/lib/scheduler.ts
+++ b/web/lib/scheduler.ts
@@ -17,6 +17,7 @@ import {
 	servers,
 	serviceRevisions,
 	services,
+	serviceCommands,
 	workQueue,
 } from "@/db/schema";
 import {
@@ -838,17 +839,38 @@ export async function cleanupStaleItems(): Promise<void> {
 
 	// Pending work is intentionally retained so commands can run when an agent
 	// reconnects. Only exhausted processing attempts are failed here.
-	const staleWorkItems = await db
-		.update(workQueue)
-		.set({ status: "failed" })
-		.where(
-			and(
-				eq(workQueue.status, "processing"),
-				lt(workQueue.startedAt, workItemLeaseThreshold),
-				sql`${workQueue.attempts} >= ${WORK_QUEUE_MAX_ATTEMPTS}`,
-			),
-		)
-		.returning({ id: workQueue.id });
+	const staleWorkItems = await db.transaction(async (tx) => {
+		const items = await tx
+			.update(workQueue)
+			.set({ status: "failed" })
+			.where(
+				and(
+					eq(workQueue.status, "processing"),
+					lt(workQueue.startedAt, workItemLeaseThreshold),
+					or(
+						eq(workQueue.type, "command"),
+						sql`${workQueue.attempts} >= ${WORK_QUEUE_MAX_ATTEMPTS}`,
+					),
+				),
+			)
+			.returning({ id: workQueue.id });
+		if (items.length > 0) {
+			await tx
+				.update(serviceCommands)
+				.set({
+					status: "failed",
+					errorMessage: "Agent did not report a result",
+					completedAt: new Date(),
+				})
+				.where(
+					inArray(
+						serviceCommands.id,
+						items.map((item) => item.id),
+					),
+				);
+		}
+		return items;
+	});
 
 	if (staleWorkItems.length > 0) {
 		console.log(
diff --git a/web/lib/service-command-retention.ts b/web/lib/service-command-retention.ts
new file mode 100644
index 00000000..ed38c560
--- /dev/null
+++ b/web/lib/service-command-retention.ts
@@ -0,0 +1,18 @@
+import { lt } from "drizzle-orm";
+import { db } from "@/db";
+import { serviceCommands } from "@/db/schema";
+import { DAY_IN_MILLISECONDS, subtractMilliseconds } from "@/lib/date";
+
+export const COMMAND_RETENTION_DAYS = 90;
+
+export async function cleanupOldServiceCommands(now = new Date()) {
+	return db
+		.delete(serviceCommands)
+		.where(
+			lt(
+				serviceCommands.createdAt,
+				subtractMilliseconds(now, COMMAND_RETENTION_DAYS * DAY_IN_MILLISECONDS),
+			),
+		)
+		.returning({ id: serviceCommands.id });
+}
diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts
index 3bd6e895..f5332b45 100644
--- a/web/lib/work-queue.ts
+++ b/web/lib/work-queue.ts
@@ -1,7 +1,13 @@
 import { randomUUID } from "node:crypto";
 import { and, eq, inArray, isNotNull, sql } from "drizzle-orm";
 import { db } from "@/db";
-import { deployments, servers, volumeBackups, workQueue } from "@/db/schema";
+import {
+	deployments,
+	servers,
+	serviceCommands,
+	volumeBackups,
+	workQueue,
+} from "@/db/schema";
 import type { WorkQueue } from "@/db/types";
 import { MINUTE_IN_MILLISECONDS, subtractMilliseconds } from "@/lib/date";
 import { inngest } from "@/lib/inngest/client";
@@ -28,6 +34,12 @@ type ReconcileWorkPayload = {
 };
 
 export type WorkPayloadByType = {
+	command: {
+		commandRunId: string;
+		deploymentId: string;
+		containerId: string;
+		command: string;
+	};
 	deploy: ReconcileWorkPayload;
 	reconcile: ReconcileWorkPayload;
 	stop: { deploymentId: string; containerId: string | null };
@@ -78,6 +90,10 @@ export type WorkItemResult = {
 	attempt: number;
 	status: "completed" | "failed";
 	error?: string;
+	output?: string;
+	exitCode?: number;
+	outputTruncated?: boolean;
+	timedOut?: boolean;
 };
 
 export type ActiveWorkItem = {
@@ -179,6 +195,19 @@ export async function completeWorkItemResults(
 	const rejected: RejectedWorkItemResult[] = [];
 
 	for (const result of results) {
+		if (
+			(result.output !== undefined &&
+				(typeof result.output !== "string" ||
+					Buffer.byteLength(result.output) > 65536)) ||
+			(result.exitCode !== undefined &&
+				(!Number.isInteger(result.exitCode) || result.exitCode < -1)) ||
+			(result.outputTruncated !== undefined &&
+				typeof result.outputTruncated !== "boolean") ||
+			(result.timedOut !== undefined && typeof result.timedOut !== "boolean")
+		) {
+			rejected.push({ id: result.id, reason: "invalid_result" });
+			continue;
+		}
 		const item = await db.transaction(async (tx) => {
 			const updated = await tx
 				.update(workQueue)
@@ -199,6 +228,23 @@ export async function completeWorkItemResults(
 			if (item.type === "restore_volume") {
 				await publishRestoreWorkResult(tx, item, result);
 			}
+			if (item.type === "command") {
+				await tx
+					.update(serviceCommands)
+					.set({
+						status: result.timedOut
+							? "timed_out"
+							: result.status === "completed" && result.exitCode === 0
+								? "succeeded"
+								: "failed",
+						output: result.output ?? "",
+						exitCode: result.exitCode,
+						outputTruncated: result.outputTruncated ?? false,
+						errorMessage: result.error ?? null,
+						completedAt: new Date(),
+					})
+					.where(eq(serviceCommands.id, item.id));
+			}
 
 			return item;
 		});
@@ -290,6 +336,12 @@ export async function claimNextWorkItem(
 
 	const row = rows[0];
 	if (!row) return null;
+	if (row.type === "command") {
+		await db
+			.update(serviceCommands)
+			.set({ status: "running", startedAt: new Date() })
+			.where(eq(serviceCommands.id, row.id));
+	}
 	if (row.type === "upgrade_agent") {
 		await markAgentUpgradeStarted(serverId, row.payload);
 	}
@@ -324,6 +376,7 @@ function claimableWorkCondition(serverId: string, staleThreshold: Date) {
 			status = 'pending'
 			OR (
 				status = 'processing'
+				AND type <> 'command'
 				AND started_at < ${staleThreshold}
 				AND attempts < ${WORK_QUEUE_MAX_ATTEMPTS}
 			)
diff --git a/web/tests/inngest-route.test.ts b/web/tests/inngest-route.test.ts
index 370fc961..316bd55d 100644
--- a/web/tests/inngest-route.test.ts
+++ b/web/tests/inngest-route.test.ts
@@ -27,6 +27,7 @@ const mocks = vi.hoisted(() => {
 		rolloutWorkflow: { id: "rollout-workflow" },
 		scheduledBackupsCheck: { id: "scheduled-backups-check" },
 		scheduledDeploymentsCheck: { id: "scheduled-deployments-check" },
+		serviceCommandRetention: { id: "service-command-retention" },
 		serviceDeletionWorkflow: { id: "service-deletion-workflow" },
 		serviceRestoreWorkflow: { id: "service-restore-workflow" },
 		staleItemsCleanup: { id: "stale-items-cleanup" },
diff --git a/web/tests/navigation.test.ts b/web/tests/navigation.test.ts
index 0e1f7a37..ebde38ce 100644
--- a/web/tests/navigation.test.ts
+++ b/web/tests/navigation.test.ts
@@ -19,7 +19,7 @@ describe("dashboard navigation catalog", () => {
 			[{ id: "server-1", name: "edge-01" }],
 		);
 
-		expect(items).toHaveLength(19);
+		expect(items).toHaveLength(20);
 		expect(items.map((item) => item.href)).toEqual(
 			expect.arrayContaining([
 				"/dashboard",
@@ -33,6 +33,7 @@ describe("dashboard navigation catalog", () => {
 				"/dashboard/projects/acme/production/services/service-1/configuration",
 				"/dashboard/projects/acme/production/services/service-1/metrics",
 				"/dashboard/projects/acme/production/services/service-1/logs",
+				"/dashboard/projects/acme/production/services/service-1/commands",
 				"/dashboard/projects/acme/production/services/service-1/requests",
 				"/dashboard/projects/acme/production/services/service-1/builds",
 				"/dashboard/projects/acme/production/services/service-1/backups",
diff --git a/web/tests/service-command-retention.test.ts b/web/tests/service-command-retention.test.ts
new file mode 100644
index 00000000..218a284e
--- /dev/null
+++ b/web/tests/service-command-retention.test.ts
@@ -0,0 +1,52 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => {
+	const returning = vi.fn().mockResolvedValue([{ id: "command-1" }]);
+	const where = vi.fn(() => ({ returning }));
+	const deleteCommand = vi.fn(() => ({ where }));
+	return {
+		db: { delete: deleteCommand },
+		deleteCommand,
+		where,
+		returning,
+		subtractMilliseconds: vi.fn(),
+	};
+});
+
+vi.mock("@/db", () => ({ db: mocks.db }));
+vi.mock("@/lib/date", () => ({
+	DAY_IN_MILLISECONDS: 86_400_000,
+	subtractMilliseconds: mocks.subtractMilliseconds,
+}));
+
+import { serviceCommands } from "@/db/schema";
+import {
+	cleanupOldServiceCommands,
+	COMMAND_RETENTION_DAYS,
+} from "@/lib/service-command-retention";
+
+describe("service command retention", () => {
+	beforeEach(() => {
+		mocks.deleteCommand.mockClear();
+		mocks.where.mockClear();
+		mocks.returning.mockClear();
+		mocks.subtractMilliseconds.mockReset();
+		mocks.subtractMilliseconds.mockReturnValue(
+			new Date("2026-05-07T00:00:00Z"),
+		);
+	});
+
+	it("deletes commands older than 90 days", async () => {
+		const now = new Date("2026-08-05T00:00:00Z");
+
+		const deleted = await cleanupOldServiceCommands(now);
+
+		expect(COMMAND_RETENTION_DAYS).toBe(90);
+		expect(mocks.subtractMilliseconds).toHaveBeenCalledWith(
+			now,
+			90 * 86_400_000,
+		);
+		expect(mocks.deleteCommand).toHaveBeenCalledWith(serviceCommands);
+		expect(deleted).toEqual([{ id: "command-1" }]);
+	});
+});
diff --git a/web/tests/service-commands-route.test.ts b/web/tests/service-commands-route.test.ts
new file mode 100644
index 00000000..064cd05d
--- /dev/null
+++ b/web/tests/service-commands-route.test.ts
@@ -0,0 +1,181 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => {
+	const queryResults: unknown[][] = [];
+
+	function createQuery(result: unknown[]) {
+		const query = {
+			from: vi.fn(() => query),
+			innerJoin: vi.fn(() => query),
+			where: vi.fn(() => query),
+			orderBy: vi.fn(() => query),
+			limit: vi.fn(() => query),
+			// oxlint-disable-next-line unicorn/no-thenable -- Drizzle query builders are awaitable.
+			then: (
+				resolve: (value: unknown[]) => unknown,
+				reject?: (reason: unknown) => unknown,
+			) => Promise.resolve(result).then(resolve, reject),
+		};
+		return query;
+	}
+
+	const insertValues = vi.fn().mockResolvedValue(undefined);
+	const tx = {
+		insert: vi.fn(() => ({ values: insertValues })),
+	};
+
+	return {
+		queryResults,
+		insertValues,
+		tx,
+		db: {
+			select: vi.fn(() => createQuery(queryResults.shift() ?? [])),
+			transaction: vi.fn(
+				async (callback: (transaction: typeof tx) => Promise<unknown>) =>
+					callback(tx),
+			),
+		},
+		requireRequestDeveloperRole: vi.fn(),
+		enqueueWork: vi.fn(),
+	};
+});
+
+vi.mock("@/db", () => ({ db: mocks.db }));
+vi.mock("@/lib/api-auth", () => ({
+	requireRequestDeveloperRole: mocks.requireRequestDeveloperRole,
+}));
+vi.mock("@/lib/work-queue", () => ({ enqueueWork: mocks.enqueueWork }));
+
+import { GET, POST } from "@/app/api/services/[id]/commands/route";
+
+const routeParams = { params: Promise.resolve({ id: "service-1" }) };
+
+function post(body: unknown) {
+	return POST(
+		new Request("http://localhost/api/services/service-1/commands", {
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify(body),
+		}),
+		routeParams,
+	);
+}
+
+describe("service commands route", () => {
+	beforeEach(() => {
+		mocks.queryResults.length = 0;
+		mocks.db.select.mockClear();
+		mocks.db.transaction.mockClear();
+		mocks.tx.insert.mockClear();
+		mocks.insertValues.mockClear();
+		mocks.enqueueWork.mockReset();
+		mocks.enqueueWork.mockResolvedValue(undefined);
+		mocks.requireRequestDeveloperRole.mockReset();
+		mocks.requireRequestDeveloperRole.mockResolvedValue({
+			ok: true,
+			session: { user: { id: "user-1", name: "Ada" } },
+		});
+	});
+
+	it("requires a developer role before reading history", async () => {
+		mocks.requireRequestDeveloperRole.mockResolvedValue({
+			ok: false,
+			response: Response.json({ error: "Forbidden" }, { status: 403 }),
+		});
+
+		const response = await GET(
+			new Request("http://localhost/api/services/service-1/commands"),
+			routeParams,
+		);
+
+		expect(response.status).toBe(403);
+		expect(mocks.db.select).not.toHaveBeenCalled();
+	});
+
+	it("rejects invalid commands before querying a target", async () => {
+		const response = await post({ deploymentId: "deployment-1", command: " " });
+
+		expect(response.status).toBe(400);
+		expect(mocks.db.select).not.toHaveBeenCalled();
+		expect(mocks.enqueueWork).not.toHaveBeenCalled();
+	});
+
+	it("stores the actor and queues the validated target transactionally", async () => {
+		mocks.queryResults.push(
+			[{ id: "service-1" }],
+			[
+				{
+					deploymentId: "deployment-1",
+					serverId: "server-1",
+					serverName: "Sydney",
+					containerId: "container-1",
+				},
+			],
+		);
+
+		const response = await post({
+			deploymentId: "deployment-1",
+			command: "printf hello",
+		});
+		const body = (await response.json()) as { id: string; status: string };
+
+		expect(response.status).toBe(202);
+		expect(body.status).toBe("pending");
+		expect(mocks.insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				id: body.id,
+				serviceId: "service-1",
+				deploymentId: "deployment-1",
+				containerId: "container-1",
+				actor: { id: "user-1", name: "Ada" },
+				command: "printf hello",
+			}),
+		);
+		expect(mocks.enqueueWork).toHaveBeenCalledWith(
+			"server-1",
+			"command",
+			{
+				commandRunId: body.id,
+				deploymentId: "deployment-1",
+				containerId: "container-1",
+				command: "printf hello",
+			},
+			{ id: body.id, tx: mocks.tx },
+		);
+	});
+
+	it("returns paginated history without internal actor IDs", async () => {
+		mocks.queryResults.push(
+			Array.from({ length: 26 }, (_, index) => ({
+				id: `command-${String(26 - index).padStart(2, "0")}`,
+				command: "whoami",
+				status: "succeeded",
+				output: "root\n",
+				exitCode: 0,
+				outputTruncated: false,
+				errorMessage: null,
+				actor: { id: "private-user-id", name: "Ada" },
+				serverName: "Sydney",
+				containerId: "container-1",
+				createdAt: new Date("2026-08-05T10:00:00Z"),
+				cursorCreatedAt: "2026-08-05 10:00:00+00",
+				startedAt: new Date("2026-08-05T10:00:01Z"),
+				completedAt: new Date("2026-08-05T10:00:02Z"),
+			})),
+		);
+
+		const response = await GET(
+			new Request("http://localhost/api/services/service-1/commands"),
+			routeParams,
+		);
+		const body = (await response.json()) as {
+			commands: Array<{ actor: Record<string, unknown> }>;
+			nextCursor: string | null;
+		};
+
+		expect(response.status).toBe(200);
+		expect(body.commands).toHaveLength(25);
+		expect(body.commands[0]?.actor).toEqual({ name: "Ada" });
+		expect(body.nextCursor).toEqual(expect.any(String));
+	});
+});
diff --git a/web/tests/work-queue.test.ts b/web/tests/work-queue.test.ts
index 51a4f678..2692d95b 100644
--- a/web/tests/work-queue.test.ts
+++ b/web/tests/work-queue.test.ts
@@ -1,4 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { PgDialect } from "drizzle-orm/pg-core";
 
 const mocks = vi.hoisted(() => {
 	const state = {
@@ -8,6 +9,7 @@ const mocks = vi.hoisted(() => {
 		persistedStatus: "processing",
 		pendingStatus: null as string | null,
 		updateMatched: false,
+		updates: [] as Record<string, unknown>[],
 	};
 
 	function createQuery(result: unknown[]) {
@@ -28,6 +30,7 @@ const mocks = vi.hoisted(() => {
 	function createUpdateQuery() {
 		const query = {
 			set: vi.fn((values: { status?: string }) => {
+				state.updates.push(values);
 				state.pendingStatus = values.status ?? null;
 				return query;
 			}),
@@ -71,6 +74,7 @@ const mocks = vi.hoisted(() => {
 				},
 			),
 			select: vi.fn(() => createQuery(state.rejectionRows)),
+			execute: vi.fn().mockResolvedValue({ rows: [] }),
 		},
 		send: vi.fn(),
 	};
@@ -109,7 +113,7 @@ vi.mock("@/lib/work-queue-notifications", () => ({
 	notifyWorkAvailable: vi.fn(),
 }));
 
-import { completeWorkItemResults } from "@/lib/work-queue";
+import { claimNextWorkItem, completeWorkItemResults } from "@/lib/work-queue";
 
 function restoreWorkItem(id: string, overrides: Record<string, unknown> = {}) {
 	return {
@@ -129,6 +133,19 @@ function restoreWorkItem(id: string, overrides: Record<string, unknown> = {}) {
 	};
 }
 
+function commandWorkItem(id: string) {
+	return {
+		id,
+		serverId: "server-1",
+		type: "command",
+		payload: JSON.stringify({ commandRunId: id }),
+		status: "completed",
+		attempts: 1,
+		createdAt: new Date(),
+		startedAt: new Date(),
+	};
+}
+
 beforeEach(() => {
 	mocks.state.updatedRows = [restoreWorkItem("work-1")];
 	mocks.state.backupRows = [{ volumeId: "volume-1", serviceId: "service-1" }];
@@ -136,14 +153,90 @@ beforeEach(() => {
 	mocks.state.persistedStatus = "processing";
 	mocks.state.pendingStatus = null;
 	mocks.state.updateMatched = false;
+	mocks.state.updates = [];
 	mocks.tx.update.mockClear();
 	mocks.tx.select.mockClear();
 	mocks.db.transaction.mockClear();
 	mocks.db.select.mockClear();
+	mocks.db.execute.mockClear();
 	mocks.send.mockReset();
 	mocks.send.mockResolvedValue(undefined);
 });
 
+describe("command work completion", () => {
+	it("does not re-lease a processing command", async () => {
+		await claimNextWorkItem("server-1");
+
+		const condition = mocks.db.execute.mock.calls[0]?.[0];
+		const query = new PgDialect().sqlToQuery(condition);
+		expect(query.sql).toContain("type <> 'command'");
+	});
+
+	it("persists successful command output", async () => {
+		mocks.state.updatedRows = [commandWorkItem("command-1")];
+
+		const result = await completeWorkItemResults("server-1", [
+			{
+				id: "command-1",
+				attempt: 1,
+				status: "completed",
+				output: "hello\n",
+				exitCode: 0,
+				outputTruncated: false,
+			},
+		]);
+
+		expect(result).toEqual({ accepted: ["command-1"], rejected: [] });
+		expect(mocks.state.updates).toContainEqual(
+			expect.objectContaining({
+				status: "succeeded",
+				output: "hello\n",
+				exitCode: 0,
+				outputTruncated: false,
+			}),
+		);
+	});
+
+	it("persists command timeouts distinctly", async () => {
+		mocks.state.updatedRows = [commandWorkItem("command-1")];
+
+		await completeWorkItemResults("server-1", [
+			{
+				id: "command-1",
+				attempt: 1,
+				status: "failed",
+				error: "command timed out after 60 seconds",
+				exitCode: 124,
+				timedOut: true,
+			},
+		]);
+
+		expect(mocks.state.updates).toContainEqual(
+			expect.objectContaining({ status: "timed_out", exitCode: 124 }),
+		);
+	});
+
+	it("rejects oversized command output without terminating the work item", async () => {
+		mocks.state.updatedRows = [commandWorkItem("command-1")];
+
+		const result = await completeWorkItemResults("server-1", [
+			{
+				id: "command-1",
+				attempt: 1,
+				status: "completed",
+				output: "x".repeat(65537),
+				exitCode: 0,
+			},
+		]);
+
+		expect(result).toEqual({
+			accepted: [],
+			rejected: [{ id: "command-1", reason: "invalid_result" }],
+		});
+		expect(mocks.db.transaction).not.toHaveBeenCalled();
+	});
+});
+
 describe("restore work completion", () => {
 	it("publishes an authorized normal restore success", async () => {
 		const result = await completeWorkItemResults("server-1", [

From 531a54db10008976d9fcf5e3ffd1d7394502e7c4 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 11:27:54 +0000
Subject: [PATCH 2/6] Use monospace font for command input

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 .../projects/[slug]/[env]/services/[serviceId]/commands/page.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
index 52e2a7aa..b4cc869f 100644
--- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
+++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
@@ -188,6 +188,7 @@ export default function CommandsPage() {
 					)}
 					<Textarea
 						aria-label="Command"
+						className="font-mono"
 						value={command}
 						onChange={(event) => setCommand(event.target.value)}
 						maxLength={4096}

From d730a69c258d07217fd114d0a803ddcb1ac8e987 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 11:30:04 +0000
Subject: [PATCH 3/6] Align command history with service lists

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 .../services/[serviceId]/commands/page.tsx    | 179 +++++++++++-------
 1 file changed, 115 insertions(+), 64 deletions(-)

diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
index b4cc869f..b839de21 100644
--- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
+++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
@@ -1,10 +1,9 @@
 "use client";
 
-import { Loader2, Terminal, XCircle } from "lucide-react";
+import { CheckCircle2, Clock, Loader2, Terminal, XCircle } from "lucide-react";
 import { useMemo, useState } from "react";
 import useSWRInfinite from "swr/infinite";
 import { useService } from "@/components/service/service-layout-client";
-import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import {
@@ -13,14 +12,27 @@ import {
 	EmptyMedia,
 	EmptyTitle,
 } from "@/components/ui/empty";
+import {
+	Item,
+	ItemContent,
+	ItemDescription,
+	ItemGroup,
+	ItemTitle,
+} from "@/components/ui/item";
 import {
 	NativeSelect,
 	NativeSelectOption,
 } from "@/components/ui/native-select";
+import { StatusBadge } from "@/components/ui/status-badge";
 import { Textarea } from "@/components/ui/textarea";
-import { formatDateTime, formatRelativeTime } from "@/lib/date";
+import {
+	formatDateTime,
+	formatElapsedDurationBetween,
+	formatRelativeTime,
+} from "@/lib/date";
 import { isObservedReady } from "@/lib/deployment-status";
 import { fetcher } from "@/lib/fetcher";
+import { cn } from "@/lib/utils";
 
 type CommandStatus =
 	| "pending"
@@ -50,26 +62,43 @@ type CommandHistory = {
 	nextCursor: string | null;
 };
 
-const STATUS_LABELS: Record<CommandStatus, string> = {
-	pending: "Queued",
-	running: "Running",
-	succeeded: "Succeeded",
-	failed: "Failed",
-	timed_out: "Timed out",
+const STATUS_CONFIG: Record<
+	CommandStatus,
+	{ label: string; icon: typeof Clock; className: string }
+> = {
+	pending: { label: "Queued", icon: Clock, className: "text-slate-500" },
+	running: { label: "Running", icon: Loader2, className: "text-blue-500" },
+	succeeded: {
+		label: "Succeeded",
+		icon: CheckCircle2,
+		className: "text-green-500",
+	},
+	failed: { label: "Failed", icon: XCircle, className: "text-red-500" },
+	timed_out: {
+		label: "Timed out",
+		icon: Clock,
+		className: "text-orange-500",
+	},
 };
 
-function CommandStatusBadge({ command }: { command: CommandRun }) {
-	const failed = command.status === "failed" || command.status === "timed_out";
+function CommandStatusBadge({
+	status,
+	className,
+	size,
+}: {
+	status: CommandStatus;
+	className?: string;
+	size?: "default" | "sm";
+}) {
+	const config = STATUS_CONFIG[status];
 	return (
-		<Badge
-			variant={failed ? "destructive" : "secondary"}
-			className={
-				command.status === "succeeded" ? "text-emerald-700" : undefined
-			}
-		>
-			{STATUS_LABELS[command.status]}
-			{command.exitCode !== null ? ` · exit ${command.exitCode}` : null}
-		</Badge>
+		<StatusBadge
+			icon={config.icon}
+			label={config.label}
+			isAnimated={status === "running"}
+			className={cn(config.className, className)}
+			size={size}
+		/>
 	);
 }
 
@@ -243,54 +272,76 @@ export default function CommandsPage() {
 						</EmptyDescription>
 					</Empty>
 				) : (
-					<div className="space-y-3">
+					<ItemGroup>
 						{history.map((item) => (
-							<Card key={item.id}>
-								<CardContent className="space-y-3 pt-6">
-									<div className="flex items-start justify-between gap-4">
-										<code className="break-all text-sm">$ {item.command}</code>
-										<CommandStatusBadge command={item} />
-									</div>
-									<p className="text-xs text-muted-foreground">
-										{item.actor.name} · {item.serverName} ·{" "}
-										{item.containerId.slice(0, 12)} ·{" "}
-										<time title={formatDateTime(item.createdAt)}>
+							<Item key={item.id} variant="outline" className="items-start">
+								<CommandStatusBadge
+									status={item.status}
+									className="w-28 max-sm:hidden"
+								/>
+								<ItemContent>
+									<ItemTitle>
+										<code
+											className="truncate font-mono text-xs"
+											title={item.command}
+										>
+											$ {item.command}
+										</code>
+									</ItemTitle>
+									<ItemDescription as="div" className="line-clamp-none">
+										<CommandStatusBadge
+											status={item.status}
+											size="sm"
+											className="mr-3 sm:hidden"
+										/>
+										<span>
+											{item.actor.name} · {item.serverName} ·{" "}
+											{item.containerId.slice(0, 12)}
+										</span>
+										<span
+											className="ml-3"
+											title={formatDateTime(item.createdAt)}
+										>
 											{formatRelativeTime(item.createdAt)}
-										</time>
-									</p>
-									{item.startedAt || item.completedAt ? (
-										<p className="text-xs text-muted-foreground">
-											{item.startedAt
-												? `Started ${formatDateTime(item.startedAt)}`
-												: null}
-											{item.startedAt && item.completedAt ? " · " : null}
-											{item.completedAt
-												? `Completed ${formatDateTime(item.completedAt)}`
+										</span>
+										{item.exitCode !== null ? (
+											<span className="ml-3">Exit: {item.exitCode}</span>
+										) : null}
+										{item.startedAt ? (
+											<span
+												className="ml-3"
+												title={`Started ${formatDateTime(item.startedAt)}${item.completedAt ? ` · Completed ${formatDateTime(item.completedAt)}` : ""}`}
+											>
+												<span className="max-sm:hidden">Duration: </span>
+												{formatElapsedDurationBetween(
+													item.startedAt,
+													item.completedAt,
+												)}
+											</span>
+										) : null}
+									</ItemDescription>
+								</ItemContent>
+								{item.output || item.errorMessage ? (
+									<details className="w-full border-t pt-2">
+										<summary className="cursor-pointer text-sm">
+											Combined output
+										</summary>
+										<pre className="mt-2 max-h-96 overflow-auto whitespace-pre-wrap rounded-md bg-muted p-3 text-xs">
+											{item.output}
+											{item.errorMessage
+												? `${item.output ? "\n" : ""}${item.errorMessage}`
 												: null}
-										</p>
-									) : null}
-									{item.output || item.errorMessage ? (
-										<details open>
-											<summary className="cursor-pointer text-sm">
-												Combined output
-											</summary>
-											<pre className="mt-2 max-h-96 overflow-auto whitespace-pre-wrap rounded-md bg-muted p-3 text-xs">
-												{item.output}
-												{item.errorMessage
-													? `${item.output ? "\n" : ""}${item.errorMessage}`
-													: null}
-											</pre>
-											{item.outputTruncated ? (
-												<p className="mt-1 text-xs text-amber-600">
-													Output was truncated at 64 KiB.
-												</p>
-											) : null}
-										</details>
-									) : null}
-								</CardContent>
-							</Card>
+										</pre>
+										{item.outputTruncated ? (
+											<p className="mt-1 text-xs text-amber-600">
+												Output was truncated at 64 KiB.
+											</p>
+										) : null}
+									</details>
+								) : null}
+							</Item>
 						))}
-					</div>
+					</ItemGroup>
 				)}
 
 				{hasMore ? (

From f184dd53f19bba76684cb1bcb9a266d9e48927aa Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 11:39:38 +0000
Subject: [PATCH 4/6] Nest typed work item results

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 agent/internal/agent/workqueue.go    |  11 +-
 agent/internal/http/client.go        |  19 +++-
 agent/internal/http/client_test.go   |  22 ++++
 web/app/api/v1/agent/status/route.ts |  12 ++-
 web/lib/work-queue.ts                | 145 ++++++++++++++++-----------
 web/tests/work-queue.test.ts         |  54 ++++++++--
 6 files changed, 185 insertions(+), 78 deletions(-)

diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go
index 2f6411c5..5f12116a 100644
--- a/agent/internal/agent/workqueue.go
+++ b/agent/internal/agent/workqueue.go
@@ -129,13 +129,16 @@ func (a *Agent) processLeasedWorkItem(item agenthttp.WorkQueueItem) {
 		Error:   errorMsg,
 	}
 	if commandResult != nil {
-		completed.Output = commandResult.Output
-		completed.ExitCode = &commandResult.ExitCode
-		completed.OutputTruncated = commandResult.Truncated
+		completed.Result = agenthttp.CommandWorkItemResult{
+			Type:            "command",
+			Output:          commandResult.Output,
+			ExitCode:        &commandResult.ExitCode,
+			OutputTruncated: commandResult.Truncated,
+			TimedOut:        commandResult.TimedOut,
+		}
 		if commandResult.TimedOut {
 			completed.Status = "failed"
 			completed.Error = "command timed out after 60 seconds"
-			completed.TimedOut = true
 		} else if commandResult.ExitCode != 0 {
 			completed.Status = "failed"
 		}
diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go
index b825e720..ecf17577 100644
--- a/agent/internal/http/client.go
+++ b/agent/internal/http/client.go
@@ -358,16 +358,27 @@ type StatusReport struct {
 }
 
 type CompletedWorkItem struct {
-	ID              string `json:"id"`
-	Attempt         int    `json:"attempt"`
-	Status          string `json:"status"`
-	Error           string `json:"error,omitempty"`
+	ID      string         `json:"id"`
+	Attempt int            `json:"attempt"`
+	Status  string         `json:"status"`
+	Error   string         `json:"error,omitempty"`
+	Result  WorkItemResult `json:"result,omitempty"`
+}
+
+type WorkItemResult interface {
+	isWorkItemResult()
+}
+
+type CommandWorkItemResult struct {
+	Type            string `json:"type"`
 	Output          string `json:"output,omitempty"`
 	ExitCode        *int   `json:"exitCode,omitempty"`
 	OutputTruncated bool   `json:"outputTruncated,omitempty"`
 	TimedOut        bool   `json:"timedOut,omitempty"`
 }
 
+func (CommandWorkItemResult) isWorkItemResult() {}
+
 type ActiveWorkItem struct {
 	ID      string `json:"id"`
 	Attempt int    `json:"attempt"`
diff --git a/agent/internal/http/client_test.go b/agent/internal/http/client_test.go
index d402012b..7a2afdaa 100644
--- a/agent/internal/http/client_test.go
+++ b/agent/internal/http/client_test.go
@@ -41,6 +41,28 @@ func TestSignedJSONRequests(t *testing.T) {
 	}
 }
 
+func TestCompletedWorkItemCommandResultJSON(t *testing.T) {
+	exitCode := 0
+	encoded, err := json.Marshal(CompletedWorkItem{
+		ID:      "command-1",
+		Attempt: 1,
+		Status:  "completed",
+		Result: CommandWorkItemResult{
+			Type:     "command",
+			Output:   "hello\n",
+			ExitCode: &exitCode,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const expected = `{"id":"command-1","attempt":1,"status":"completed","result":{"type":"command","output":"hello\n","exitCode":0}}`
+	if string(encoded) != expected {
+		t.Fatalf("unexpected completion JSON: %s", encoded)
+	}
+}
+
 func TestUpdateBuildStatusImageURI(t *testing.T) {
 	keyPair, err := crypto.GenerateKeyPair()
 	if err != nil {
diff --git a/web/app/api/v1/agent/status/route.ts b/web/app/api/v1/agent/status/route.ts
index daf4c992..f29b4e47 100644
--- a/web/app/api/v1/agent/status/route.ts
+++ b/web/app/api/v1/agent/status/route.ts
@@ -5,13 +5,13 @@ import {
 	type ActiveWorkItem,
 	claimNextWorkItem,
 	completeWorkItemResults,
+	type CompletedWorkItem,
 	renewActiveWorkItems,
-	type WorkItemResult,
 } from "@/lib/work-queue";
 
 type StatusRequestBody = {
 	statusReport?: StatusReport;
-	completedWorkItems?: WorkItemResult[];
+	completedWorkItems?: CompletedWorkItem[];
 	activeWorkItems?: ActiveWorkItem[];
 	serverlessTransitions?: unknown[];
 };
@@ -76,16 +76,18 @@ export async function POST(request: NextRequest) {
 	});
 }
 
-function isValidWorkItemResult(value: unknown): value is WorkItemResult {
+function isValidWorkItemResult(value: unknown): value is CompletedWorkItem {
 	if (!value || typeof value !== "object") return false;
 
-	const candidate = value as WorkItemResult;
+	const candidate = value as CompletedWorkItem;
 	return (
 		typeof candidate.id === "string" &&
 		Number.isInteger(candidate.attempt) &&
 		candidate.attempt > 0 &&
 		(candidate.status === "completed" || candidate.status === "failed") &&
-		(candidate.error === undefined || typeof candidate.error === "string")
+		(candidate.error === undefined || typeof candidate.error === "string") &&
+		(candidate.result === undefined ||
+			(Boolean(candidate.result) && typeof candidate.result === "object"))
 	);
 }
 
diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts
index f5332b45..bcd71432 100644
--- a/web/lib/work-queue.ts
+++ b/web/lib/work-queue.ts
@@ -86,16 +86,21 @@ export type WorkPayloadByType = {
 };
 
 export type WorkItemResult = {
-	id: string;
-	attempt: number;
-	status: "completed" | "failed";
-	error?: string;
+	type: "command";
 	output?: string;
 	exitCode?: number;
 	outputTruncated?: boolean;
 	timedOut?: boolean;
 };
 
+export type CompletedWorkItem = {
+	id: string;
+	attempt: number;
+	status: "completed" | "failed";
+	error?: string;
+	result?: WorkItemResult;
+};
+
 export type ActiveWorkItem = {
 	id: string;
 	attempt: number;
@@ -186,7 +191,7 @@ export async function enqueueRegistrySyncForAllRegisteredServers(
 
 export async function completeWorkItemResults(
 	serverId: string,
-	results: WorkItemResult[],
+	results: CompletedWorkItem[],
 ): Promise<{
 	accepted: string[];
 	rejected: RejectedWorkItemResult[];
@@ -195,59 +200,66 @@ export async function completeWorkItemResults(
 	const rejected: RejectedWorkItemResult[] = [];
 
 	for (const result of results) {
-		if (
-			(result.output !== undefined &&
-				(typeof result.output !== "string" ||
-					Buffer.byteLength(result.output) > 65536)) ||
-			(result.exitCode !== undefined &&
-				(!Number.isInteger(result.exitCode) || result.exitCode < -1)) ||
-			(result.outputTruncated !== undefined &&
-				typeof result.outputTruncated !== "boolean") ||
-			(result.timedOut !== undefined && typeof result.timedOut !== "boolean")
-		) {
+		if (!isValidWorkItemResult(result.result)) {
 			rejected.push({ id: result.id, reason: "invalid_result" });
 			continue;
 		}
-		const item = await db.transaction(async (tx) => {
-			const updated = await tx
-				.update(workQueue)
-				.set({ status: result.status })
-				.where(
-					and(
-						eq(workQueue.id, result.id),
-						eq(workQueue.serverId, serverId),
-						eq(workQueue.status, "processing"),
-						eq(workQueue.attempts, result.attempt),
-					),
-				)
-				.returning();
-
-			const item = updated[0];
-			if (!item) return null;
-
-			if (item.type === "restore_volume") {
-				await publishRestoreWorkResult(tx, item, result);
-			}
-			if (item.type === "command") {
-				await tx
-					.update(serviceCommands)
-					.set({
-						status: result.timedOut
-							? "timed_out"
-							: result.status === "completed" && result.exitCode === 0
-								? "succeeded"
-								: "failed",
-						output: result.output ?? "",
-						exitCode: result.exitCode,
-						outputTruncated: result.outputTruncated ?? false,
-						errorMessage: result.error ?? null,
-						completedAt: new Date(),
-					})
-					.where(eq(serviceCommands.id, item.id));
+		let item: WorkQueue | null;
+		try {
+			item = await db.transaction(async (tx) => {
+				const updated = await tx
+					.update(workQueue)
+					.set({ status: result.status })
+					.where(
+						and(
+							eq(workQueue.id, result.id),
+							eq(workQueue.serverId, serverId),
+							eq(workQueue.status, "processing"),
+							eq(workQueue.attempts, result.attempt),
+						),
+					)
+					.returning();
+
+				const item = updated[0];
+				if (!item) return null;
+
+				if (result.result && item.type !== result.result.type) {
+					throw new WorkItemResultTypeMismatchError();
+				}
+				if (item.type === "restore_volume") {
+					await publishRestoreWorkResult(tx, item, result);
+				}
+				if (item.type === "command") {
+					const commandResult = result.result;
+					if (result.status === "completed" && !commandResult) {
+						throw new WorkItemResultTypeMismatchError();
+					}
+					await tx
+						.update(serviceCommands)
+						.set({
+							status: commandResult?.timedOut
+								? "timed_out"
+								: result.status === "completed" && commandResult?.exitCode === 0
+									? "succeeded"
+									: "failed",
+							output: commandResult?.output ?? "",
+							exitCode: commandResult?.exitCode,
+							outputTruncated: commandResult?.outputTruncated ?? false,
+							errorMessage: result.error ?? null,
+							completedAt: new Date(),
+						})
+						.where(eq(serviceCommands.id, item.id));
+				}
+
+				return item;
+			});
+		} catch (error) {
+			if (error instanceof WorkItemResultTypeMismatchError) {
+				rejected.push({ id: result.id, reason: "invalid_result" });
+				continue;
 			}
-
-			return item;
-		});
+			throw error;
+		}
 
 		if (!item) {
 			rejected.push({
@@ -266,6 +278,23 @@ export async function completeWorkItemResults(
 	return { accepted, rejected };
 }
 
+class WorkItemResultTypeMismatchError extends Error {}
+
+function isValidWorkItemResult(result: WorkItemResult | undefined): boolean {
+	if (result === undefined) return true;
+	return (
+		result.type === "command" &&
+		(result.output === undefined ||
+			(typeof result.output === "string" &&
+				Buffer.byteLength(result.output) <= 65536)) &&
+		(result.exitCode === undefined ||
+			(Number.isInteger(result.exitCode) && result.exitCode >= -1)) &&
+		(result.outputTruncated === undefined ||
+			typeof result.outputTruncated === "boolean") &&
+		(result.timedOut === undefined || typeof result.timedOut === "boolean")
+	);
+}
+
 export async function renewActiveWorkItems(
 	serverId: string,
 	items: ActiveWorkItem[],
@@ -435,7 +464,7 @@ async function getRejectionReason(
 async function publishRestoreWorkResult(
 	tx: WorkQueueTransaction,
 	item: WorkQueue,
-	result: WorkItemResult,
+	result: CompletedWorkItem,
 ): Promise<void> {
 	let value: unknown;
 	try {
@@ -545,7 +574,7 @@ async function publishRestoreWorkResult(
 
 async function runWorkItemCompletionSideEffects(
 	item: WorkQueue,
-	result: WorkItemResult,
+	result: CompletedWorkItem,
 ): Promise<void> {
 	if (item.type === "force_cleanup" && item.payload) {
 		await runForceCleanupCompletionSideEffects(item, result);
@@ -613,7 +642,7 @@ async function runWorkItemCompletionSideEffects(
 
 async function runAgentUpgradeCompletionSideEffects(
 	item: WorkQueue,
-	result: WorkItemResult,
+	result: CompletedWorkItem,
 ): Promise<void> {
 	try {
 		const payload = JSON.parse(item.payload) as { targetVersion?: string };
@@ -667,7 +696,7 @@ async function runAgentUpgradeCompletionSideEffects(
 
 async function runForceCleanupCompletionSideEffects(
 	item: WorkQueue,
-	result: WorkItemResult,
+	result: CompletedWorkItem,
 ): Promise<void> {
 	try {
 		const payload = JSON.parse(item.payload) as {
diff --git a/web/tests/work-queue.test.ts b/web/tests/work-queue.test.ts
index 2692d95b..8cd2e66c 100644
--- a/web/tests/work-queue.test.ts
+++ b/web/tests/work-queue.test.ts
@@ -180,9 +180,12 @@ describe("command work completion", () => {
 				id: "command-1",
 				attempt: 1,
 				status: "completed",
-				output: "hello\n",
-				exitCode: 0,
-				outputTruncated: false,
+				result: {
+					type: "command",
+					output: "hello\n",
+					exitCode: 0,
+					outputTruncated: false,
+				},
 			},
 		]);
 
@@ -206,8 +209,11 @@ describe("command work completion", () => {
 				attempt: 1,
 				status: "failed",
 				error: "command timed out after 60 seconds",
-				exitCode: 124,
-				timedOut: true,
+				result: {
+					type: "command",
+					exitCode: 124,
+					timedOut: true,
+				},
 			},
 		]);
 
@@ -224,8 +230,11 @@ describe("command work completion", () => {
 				id: "command-1",
 				attempt: 1,
 				status: "completed",
-				output: "x".repeat(65537),
-				exitCode: 0,
+				result: {
+					type: "command",
+					output: "x".repeat(65537),
+					exitCode: 0,
+				},
 			},
 		]);
 
@@ -235,6 +244,37 @@ describe("command work completion", () => {
 		});
 		expect(mocks.db.transaction).not.toHaveBeenCalled();
 	});
+
+	it("rejects a successful command completion without a typed result", async () => {
+		mocks.state.updatedRows = [commandWorkItem("command-1")];
+
+		const result = await completeWorkItemResults("server-1", [
+			{ id: "command-1", attempt: 1, status: "completed" },
+		]);
+
+		expect(result).toEqual({
+			accepted: [],
+			rejected: [{ id: "command-1", reason: "invalid_result" }],
+		});
+		expect(mocks.state.persistedStatus).toBe("processing");
+	});
+
+	it("rejects command result data for a different work type", async () => {
+		const result = await completeWorkItemResults("server-1", [
+			{
+				id: "work-1",
+				attempt: 1,
+				status: "failed",
+				result: { type: "command", exitCode: 1 },
+			},
+		]);
+
+		expect(result).toEqual({
+			accepted: [],
+			rejected: [{ id: "work-1", reason: "invalid_result" }],
+		});
+		expect(mocks.state.persistedStatus).toBe("processing");
+	});
 });
 
 describe("restore work completion", () => {

From 55211e839f0df9fc6953b5d8ea296837d8116039 Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 12:12:14 +0000
Subject: [PATCH 5/6] Harden container command execution

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 agent/README.md                               |   4 +-
 agent/internal/agent/handlers.go              |   5 +-
 agent/internal/agent/handlers_command_test.go |  26 ++
 agent/internal/container/libpod_exec.go       | 366 ++++++++++++++++++
 agent/internal/container/libpod_exec_test.go  | 301 ++++++++++++++
 agent/internal/container/runtime.go           |  41 --
 agent/internal/container/runtime_test.go      |  44 ---
 agent/internal/container/stats.go             |  18 +-
 agent/internal/container/stats_test.go        |  12 +-
 .../services/[serviceId]/commands/page.tsx    |   2 +-
 web/app/api/services/[id]/commands/route.ts   |   1 +
 web/lib/service-command-retention.ts          |   7 +-
 web/lib/work-queue.ts                         |   1 +
 web/tests/service-command-retention.test.ts   |   9 +-
 web/tests/service-commands-route.test.ts      |   2 +
 15 files changed, 726 insertions(+), 113 deletions(-)
 create mode 100644 agent/internal/agent/handlers_command_test.go
 create mode 100644 agent/internal/container/libpod_exec.go
 create mode 100644 agent/internal/container/libpod_exec_test.go

diff --git a/agent/README.md b/agent/README.md
index a4a0cb15..5c999d4a 100644
--- a/agent/README.md
+++ b/agent/README.md
@@ -15,7 +15,7 @@ The agent supports two modes:
 
 ### All Nodes
 - WireGuard (`wg` and `wg-quick` commands)
-- Podman
+- Podman 4.8 or newer (required for command execution cleanup)
 - BuildKit + buildctl
 - Railpack
 
@@ -226,7 +226,7 @@ WantedBy=multi-user.target
 ```
 
 `KillMode=process` ensures only the agent process is killed on restart, not container processes.
-The rootful Podman API socket at `/run/podman/podman.sock` is required for container metrics collection.
+The rootful Podman API socket at `/run/podman/podman.sock` is required for container metrics collection and command execution. Command timeouts force-remove the foreground exec session; deliberately backgrounded descendants are not guaranteed to be removed.
 
 ```bash
 sudo systemctl daemon-reload
diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go
index 3686144d..1ffbf8a8 100644
--- a/agent/internal/agent/handlers.go
+++ b/agent/internal/agent/handlers.go
@@ -23,6 +23,7 @@ import (
 func (a *Agent) ProcessCommand(item agenthttp.WorkQueueItem) (container.CommandResult, error) {
 	var payload struct {
 		CommandRunID string `json:"commandRunId"`
+		ServiceID    string `json:"serviceId"`
 		DeploymentID string `json:"deploymentId"`
 		ContainerID  string `json:"containerId"`
 		Command      string `json:"command"`
@@ -30,10 +31,10 @@ func (a *Agent) ProcessCommand(item agenthttp.WorkQueueItem) (container.CommandR
 	if err := json.Unmarshal([]byte(item.Payload), &payload); err != nil {
 		return container.CommandResult{}, fmt.Errorf("failed to parse command payload: %w", err)
 	}
-	if payload.CommandRunID != item.ID || payload.DeploymentID == "" || payload.ContainerID == "" || payload.Command == "" || utf8.RuneCountInString(payload.Command) > 4096 {
+	if payload.CommandRunID != item.ID || payload.ServiceID == "" || payload.DeploymentID == "" || payload.ContainerID == "" || payload.Command == "" || utf8.RuneCountInString(payload.Command) > 4096 {
 		return container.CommandResult{}, fmt.Errorf("invalid command payload")
 	}
-	return container.ExecCommand(payload.ContainerID, payload.Command)
+	return container.ExecCommand(payload.ContainerID, payload.ServiceID, payload.DeploymentID, payload.Command)
 }
 
 func (a *Agent) ProcessRestart(item agenthttp.WorkQueueItem) error {
diff --git a/agent/internal/agent/handlers_command_test.go b/agent/internal/agent/handlers_command_test.go
new file mode 100644
index 00000000..f7e3efcc
--- /dev/null
+++ b/agent/internal/agent/handlers_command_test.go
@@ -0,0 +1,26 @@
+package agent
+
+import (
+	"testing"
+
+	agenthttp "techulus/cloud-agent/internal/http"
+)
+
+func TestProcessCommandRequiresOwnershipIdentifiers(t *testing.T) {
+	tests := []struct {
+		name    string
+		payload string
+	}{
+		{"service ID", `{"commandRunId":"run","deploymentId":"deployment","containerId":"container","command":"true"}`},
+		{"deployment ID", `{"commandRunId":"run","serviceId":"service","containerId":"container","command":"true"}`},
+		{"container ID", `{"commandRunId":"run","serviceId":"service","deploymentId":"deployment","command":"true"}`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := (&Agent{}).ProcessCommand(agenthttp.WorkQueueItem{ID: "run", Payload: tt.payload})
+			if err == nil || err.Error() != "invalid command payload" {
+				t.Fatalf("expected invalid payload for missing %s, got %v", tt.name, err)
+			}
+		})
+	}
+}
diff --git a/agent/internal/container/libpod_exec.go b/agent/internal/container/libpod_exec.go
new file mode 100644
index 00000000..44089ca5
--- /dev/null
+++ b/agent/internal/container/libpod_exec.go
@@ -0,0 +1,366 @@
+package container
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	libpodExecBasePath = "/v4.8.0/libpod"
+	libpodBodyLimit    = 1024 * 1024
+	libpodErrorLimit   = 4 * 1024
+	libpodRequestTime  = 10 * time.Second
+)
+
+type execContainerInspect struct {
+	ID    string `json:"Id"`
+	State struct {
+		Running bool `json:"Running"`
+	} `json:"State"`
+	Config struct {
+		Labels map[string]string `json:"Labels"`
+	} `json:"Config"`
+}
+
+type execCreateResponse struct {
+	ID string `json:"Id"`
+}
+
+type execInspectResponse struct {
+	Running  bool `json:"Running"`
+	ExitCode int  `json:"ExitCode"`
+}
+
+type attachedExecStream struct {
+	io.Reader
+	closers []io.Closer
+}
+
+func (s *attachedExecStream) Close() error {
+	var closeErrors []error
+	for _, closer := range s.closers {
+		if err := closer.Close(); err != nil {
+			closeErrors = append(closeErrors, err)
+		}
+	}
+	return errors.Join(closeErrors...)
+}
+
+func ExecCommand(containerID, serviceID, deploymentID, command string) (CommandResult, error) {
+	if err := execPreflight(); err != nil {
+		return CommandResult{}, err
+	}
+	if err := verifyExecContainer(containerID, serviceID, deploymentID); err != nil {
+		return CommandResult{}, err
+	}
+
+	execID, err := createExec(containerID, command)
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	startCtx, cancelStart := context.WithTimeout(context.Background(), libpodRequestTime)
+	stream, err := startAttachedExec(startCtx, execID)
+	cancelStart()
+	if err != nil {
+		cleanupErr := removeExecSession(execID, true)
+		if cleanupErr != nil {
+			return CommandResult{}, fmt.Errorf("failed to start exec session: %w; cleanup failed: %v", err, cleanupErr)
+		}
+		return CommandResult{}, fmt.Errorf("failed to start exec session: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
+	defer cancel()
+
+	var output limitedBuffer
+	streamDone := make(chan error, 1)
+	go func() {
+		streamDone <- copyMultiplexedOutput(stream, &output)
+	}()
+
+	select {
+	case streamErr := <-streamDone:
+		_ = stream.Close()
+		if streamErr != nil {
+			if cleanupErr := removeExecSession(execID, true); cleanupErr != nil {
+				return commandResult(&output), fmt.Errorf("failed to read exec output: %w; cleanup failed: %v", streamErr, cleanupErr)
+			}
+			return commandResult(&output), fmt.Errorf("failed to read exec output: %w", streamErr)
+		}
+		inspect, err := inspectExecSession(execID)
+		if err != nil {
+			_ = removeExecSession(execID, true)
+			return commandResult(&output), err
+		}
+		if inspect.Running {
+			_ = removeExecSession(execID, true)
+			return commandResult(&output), fmt.Errorf("exec session remained running after attached stream ended")
+		}
+		if err := removeExecSession(execID, false); err != nil {
+			return commandResult(&output), err
+		}
+		result := commandResult(&output)
+		result.ExitCode = inspect.ExitCode
+		return result, nil
+	case <-ctx.Done():
+		_ = stream.Close()
+		if err := removeExecSession(execID, true); err != nil {
+			return commandResult(&output), fmt.Errorf("command deadline exceeded but exec session cleanup failed: %w", err)
+		}
+		select {
+		case <-streamDone:
+		case <-time.After(time.Second):
+		}
+		result := commandResult(&output)
+		result.ExitCode = 124
+		result.TimedOut = true
+		return result, nil
+	}
+}
+
+func commandResult(output *limitedBuffer) CommandResult {
+	text, truncated := output.snapshot()
+	text = strings.ToValidUTF8(text, "�")
+	if len(text) > CommandOutputLimit {
+		text = text[:CommandOutputLimit]
+		for !utf8.ValidString(text) {
+			text = text[:len(text)-1]
+		}
+		truncated = true
+	}
+	return CommandResult{Output: text, Truncated: truncated}
+}
+
+func execPreflight() error {
+	ctx, cancel := context.WithTimeout(context.Background(), libpodRequestTime)
+	defer cancel()
+	resp, err := libpodRequest(ctx, http.MethodGet, libpodExecBasePath+"/version", nil)
+	if err != nil {
+		return fmt.Errorf("Podman 4.8 exec API preflight failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if err := requireSuccess(resp, "Podman 4.8 exec API preflight"); err != nil {
+		return err
+	}
+	_, err = io.Copy(io.Discard, io.LimitReader(resp.Body, libpodBodyLimit+1))
+	return err
+}
+
+func verifyExecContainer(containerID, serviceID, deploymentID string) error {
+	var inspect execContainerInspect
+	if err := libpodJSON(http.MethodGet, libpodExecBasePath+"/containers/"+url.PathEscape(containerID)+"/json", nil, &inspect); err != nil {
+		return fmt.Errorf("failed to inspect command container: %w", err)
+	}
+	if inspect.ID != containerID {
+		return fmt.Errorf("inspected container ID does not match command target")
+	}
+	if !inspect.State.Running {
+		return fmt.Errorf("container is not running")
+	}
+	if inspect.Config.Labels["techulus.service.id"] != serviceID || inspect.Config.Labels["techulus.deployment.id"] != deploymentID {
+		return fmt.Errorf("container ownership does not match command target")
+	}
+	return nil
+}
+
+func createExec(containerID, command string) (string, error) {
+	body := struct {
+		AttachStdout bool     `json:"AttachStdout"`
+		AttachStderr bool     `json:"AttachStderr"`
+		TTY          bool     `json:"Tty"`
+		Cmd          []string `json:"Cmd"`
+	}{true, true, false, []string{"/bin/sh", "-c", command}}
+	var created execCreateResponse
+	if err := libpodJSON(http.MethodPost, libpodExecBasePath+"/containers/"+url.PathEscape(containerID)+"/exec", body, &created); err != nil {
+		return "", fmt.Errorf("failed to create exec session: %w", err)
+	}
+	if created.ID == "" {
+		return "", fmt.Errorf("failed to create exec session: Podman returned an empty ID")
+	}
+	return created.ID, nil
+}
+
+func inspectExecSession(execID string) (execInspectResponse, error) {
+	var inspect execInspectResponse
+	err := libpodJSON(http.MethodGet, libpodExecBasePath+"/exec/"+url.PathEscape(execID)+"/json", nil, &inspect)
+	if err != nil {
+		return inspect, fmt.Errorf("failed to inspect exec session: %w", err)
+	}
+	return inspect, nil
+}
+
+func removeExecSession(execID string, force bool) error {
+	body := struct {
+		Force bool `json:"Force"`
+	}{force}
+	if err := libpodJSON(http.MethodPost, libpodExecBasePath+"/exec/"+url.PathEscape(execID)+"/remove", body, nil); err != nil {
+		return fmt.Errorf("failed to remove exec session: %w", err)
+	}
+	return nil
+}
+
+func libpodJSON(method, path string, requestBody, responseBody any) error {
+	ctx, cancel := context.WithTimeout(context.Background(), libpodRequestTime)
+	defer cancel()
+	var body io.Reader
+	if requestBody != nil {
+		encoded, err := json.Marshal(requestBody)
+		if err != nil {
+			return err
+		}
+		body = bytes.NewReader(encoded)
+	}
+	resp, err := libpodRequest(ctx, method, path, body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if err := requireSuccess(resp, "Podman request"); err != nil {
+		return err
+	}
+	if responseBody == nil {
+		data, err := io.ReadAll(io.LimitReader(resp.Body, libpodBodyLimit+1))
+		if err != nil {
+			return err
+		}
+		if len(data) > libpodBodyLimit {
+			return fmt.Errorf("Podman response exceeds limit")
+		}
+		return nil
+	}
+	limited := io.LimitReader(resp.Body, libpodBodyLimit+1)
+	data, err := io.ReadAll(limited)
+	if err != nil {
+		return err
+	}
+	if len(data) > libpodBodyLimit {
+		return fmt.Errorf("Podman response exceeds limit")
+	}
+	if err := json.Unmarshal(data, responseBody); err != nil {
+		return fmt.Errorf("invalid Podman response: %w", err)
+	}
+	return nil
+}
+
+func libpodRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, method, podmanBaseURL+path, body)
+	if err != nil {
+		return nil, err
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	return podmanHTTPClient.Do(req)
+}
+
+func requireSuccess(resp *http.Response, operation string) error {
+	if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
+		return nil
+	}
+	message, _ := io.ReadAll(io.LimitReader(resp.Body, libpodErrorLimit))
+	return fmt.Errorf("%s failed: Podman returned %s: %s", operation, resp.Status, strings.TrimSpace(string(message)))
+}
+
+func startAttachedExec(ctx context.Context, execID string) (io.ReadCloser, error) {
+	conn, err := podmanDialContext(ctx, "unix", "podman")
+	if err != nil {
+		return nil, err
+	}
+	success := false
+	defer func() {
+		if !success {
+			_ = conn.Close()
+		}
+	}()
+	if deadline, ok := ctx.Deadline(); ok {
+		_ = conn.SetDeadline(deadline)
+	}
+	body := strings.NewReader(`{"Detach":false,"Tty":false}`)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, podmanBaseURL+libpodExecBasePath+"/exec/"+url.PathEscape(execID)+"/start", body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Connection", "Upgrade")
+	req.Header.Set("Upgrade", "tcp")
+	if err := req.Write(conn); err != nil {
+		return nil, err
+	}
+	buffered := bufio.NewReader(conn)
+	resp, err := http.ReadResponse(buffered, req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusSwitchingProtocols && resp.StatusCode != http.StatusOK {
+		defer resp.Body.Close()
+		return nil, requireSuccess(resp, "start attached exec session")
+	}
+	if err := conn.SetDeadline(time.Time{}); err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
+	success = true
+	if resp.StatusCode == http.StatusSwitchingProtocols {
+		return &attachedExecStream{Reader: buffered, closers: []io.Closer{conn}}, nil
+	}
+	return &attachedExecStream{
+		Reader:  resp.Body,
+		closers: []io.Closer{resp.Body, conn},
+	}, nil
+}
+
+func copyMultiplexedOutput(reader io.Reader, output io.Writer) error {
+	var header [8]byte
+	buffer := make([]byte, 32*1024)
+	for {
+		_, err := io.ReadFull(reader, header[:])
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("invalid multiplexed frame header: %w", err)
+		}
+		if header[1] != 0 || header[2] != 0 || header[3] != 0 {
+			return fmt.Errorf("invalid multiplexed frame header")
+		}
+		remaining := uint64(binary.BigEndian.Uint32(header[4:]))
+		if header[0] == 3 {
+			messageLength := min(remaining, uint64(libpodErrorLimit))
+			message := make([]byte, messageLength)
+			if _, err := io.ReadFull(reader, message); err != nil {
+				return fmt.Errorf("invalid multiplexed error frame: %w", err)
+			}
+			if _, err := io.CopyN(io.Discard, reader, int64(remaining-messageLength)); err != nil {
+				return fmt.Errorf("invalid multiplexed error frame: %w", err)
+			}
+			return fmt.Errorf("Podman exec stream failed: %s", strings.TrimSpace(string(message)))
+		}
+		if header[0] != 0 && header[0] != 1 && header[0] != 2 {
+			return fmt.Errorf("invalid multiplexed stream %d", header[0])
+		}
+		for remaining > 0 {
+			chunk := uint64(len(buffer))
+			if remaining < chunk {
+				chunk = remaining
+			}
+			if _, err := io.ReadFull(reader, buffer[:int(chunk)]); err != nil {
+				return fmt.Errorf("invalid multiplexed frame payload: %w", err)
+			}
+			if _, err := output.Write(buffer[:int(chunk)]); err != nil {
+				return err
+			}
+			remaining -= chunk
+		}
+	}
+}
diff --git a/agent/internal/container/libpod_exec_test.go b/agent/internal/container/libpod_exec_test.go
new file mode 100644
index 00000000..ad00d0c2
--- /dev/null
+++ b/agent/internal/container/libpod_exec_test.go
@@ -0,0 +1,301 @@
+package container
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCopyMultiplexedOutputRejectsMalformedFrames(t *testing.T) {
+	var output limitedBuffer
+	if err := copyMultiplexedOutput(strings.NewReader("short"), &output); err == nil {
+		t.Fatal("expected truncated frame header to fail")
+	}
+
+	var frame bytes.Buffer
+	var header [8]byte
+	header[0] = 9
+	binary.BigEndian.PutUint32(header[4:], 1)
+	frame.Write(header[:])
+	frame.WriteByte('x')
+	if err := copyMultiplexedOutput(&frame, &output); err == nil {
+		t.Fatal("expected unknown stream to fail")
+	}
+}
+
+func TestCopyMultiplexedOutputBoundsPodmanErrors(t *testing.T) {
+	var frame bytes.Buffer
+	var header [8]byte
+	header[0] = 3
+	binary.BigEndian.PutUint32(header[4:], 10_000)
+	frame.Write(header[:])
+	frame.WriteString(strings.Repeat("x", 10_000))
+
+	err := copyMultiplexedOutput(&frame, &limitedBuffer{})
+	if err == nil || len(err.Error()) > libpodErrorLimit+100 {
+		t.Fatalf("unexpected bounded stream error: length=%d err=%v", len(err.Error()), err)
+	}
+}
+
+func TestLibpodJSONRejectsOversizedResponses(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(strings.Repeat("x", libpodBodyLimit+1)))
+	}))
+	defer server.Close()
+	oldClient, oldBase := podmanHTTPClient, podmanBaseURL
+	podmanHTTPClient, podmanBaseURL = server.Client(), server.URL
+	t.Cleanup(func() { podmanHTTPClient, podmanBaseURL = oldClient, oldBase })
+
+	if err := libpodJSON(http.MethodGet, "/oversized", nil, &struct{}{}); err == nil || !strings.Contains(err.Error(), "exceeds limit") {
+		t.Fatalf("expected oversized response rejection, got %v", err)
+	}
+}
+
+func TestExecCommandPreflightFailsBeforeCreate(t *testing.T) {
+	api := installFakeExecAPI(t)
+	api.preflightStatus = http.StatusNotFound
+	if _, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "true"); err == nil {
+		t.Fatal("expected old API rejection")
+	}
+	if api.createCount != 0 {
+		t.Fatalf("created %d exec sessions after failed preflight", api.createCount)
+	}
+}
+
+func TestExecCommandRejectsInvalidOwnershipBeforeCreate(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(*fakeExecAPI)
+	}{
+		{"service mismatch", func(api *fakeExecAPI) { api.inspectServiceID = "other" }},
+		{"deployment mismatch", func(api *fakeExecAPI) { api.inspectDeploymentID = "other" }},
+		{"unmanaged", func(api *fakeExecAPI) { api.inspectServiceID, api.inspectDeploymentID = "", "" }},
+		{"stopped", func(api *fakeExecAPI) { api.running = false }},
+		{"ID mismatch", func(api *fakeExecAPI) { api.inspectContainerID = "other" }},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			api := installFakeExecAPI(t)
+			tt.mutate(api)
+			if _, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "true"); err == nil {
+				t.Fatal("expected rejection")
+			}
+			if api.createCount != 0 {
+				t.Fatal("exec session was created")
+			}
+		})
+	}
+}
+
+func TestExecCommandLifecycleAndOutput(t *testing.T) {
+	tests := []struct {
+		name      string
+		output    string
+		exitCode  int
+		truncated bool
+	}{
+		{"multiplexed output", "stdoutstderr", 0, false},
+		{"nonzero exit", "bad", 7, false},
+		{"exact limit", strings.Repeat("x", CommandOutputLimit), 0, false},
+		{"truncated", strings.Repeat("x", CommandOutputLimit+1), 0, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			api := installFakeExecAPI(t)
+			api.output, api.exitCode = tt.output, tt.exitCode
+			result, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "printf test")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if result.ExitCode != tt.exitCode || result.Truncated != tt.truncated || result.TimedOut {
+				t.Fatalf("unexpected result: %+v", result)
+			}
+			if result.Output != tt.output[:min(len(tt.output), CommandOutputLimit)] {
+				t.Fatalf("unexpected output length/content: %d", len(result.Output))
+			}
+			if api.removeCount != 1 || api.lastRemoveForce {
+				t.Fatalf("normal removal = count %d force %v", api.removeCount, api.lastRemoveForce)
+			}
+			if api.createdCommand != "printf test" {
+				t.Fatalf("created command = %q", api.createdCommand)
+			}
+			if !api.startRequestValid {
+				t.Fatal("attached exec start options were invalid")
+			}
+		})
+	}
+}
+
+func TestExecCommandSupportsAttachedOKResponse(t *testing.T) {
+	api := installFakeExecAPI(t)
+	api.startStatus = http.StatusOK
+	api.output = "ok response"
+
+	result, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "true")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Output != api.output || result.ExitCode != 0 {
+		t.Fatalf("unexpected result: %+v", result)
+	}
+}
+
+func TestExecCommandTimeoutForceRemoves(t *testing.T) {
+	api := installFakeExecAPI(t)
+	api.hang = true
+	oldTimeout := commandTimeout
+	commandTimeout = 20 * time.Millisecond
+	t.Cleanup(func() { commandTimeout = oldTimeout })
+	result, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "sleep")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !result.TimedOut || result.ExitCode != 124 || api.removeCount != 1 || !api.lastRemoveForce {
+		t.Fatalf("unexpected timeout result=%+v remove=%d force=%v", result, api.removeCount, api.lastRemoveForce)
+	}
+}
+
+func TestExecCommandForceRemoveFailureIsNotTimeout(t *testing.T) {
+	api := installFakeExecAPI(t)
+	api.hang = true
+	api.removeStatus = http.StatusInternalServerError
+	oldTimeout := commandTimeout
+	commandTimeout = 20 * time.Millisecond
+	t.Cleanup(func() { commandTimeout = oldTimeout })
+	result, err := ExecCommand(api.containerID, api.serviceID, api.deploymentID, "sleep")
+	if err == nil || result.TimedOut {
+		t.Fatalf("result=%+v err=%v", result, err)
+	}
+}
+
+type fakeExecAPI struct {
+	server              *httptest.Server
+	containerID         string
+	serviceID           string
+	deploymentID        string
+	inspectContainerID  string
+	inspectServiceID    string
+	inspectDeploymentID string
+	running             bool
+	preflightStatus     int
+	startStatus         int
+	removeStatus        int
+	output              string
+	exitCode            int
+	hang                bool
+	createCount         int
+	removeCount         int
+	lastRemoveForce     bool
+	createdCommand      string
+	startRequestValid   bool
+	stopOnce            sync.Once
+	stop                chan struct{}
+}
+
+func installFakeExecAPI(t *testing.T) *fakeExecAPI {
+	t.Helper()
+	api := &fakeExecAPI{
+		containerID: "container/id", serviceID: "service", deploymentID: "deployment",
+		inspectContainerID: "container/id", inspectServiceID: "service", inspectDeploymentID: "deployment",
+		running: true, preflightStatus: http.StatusOK, startStatus: http.StatusSwitchingProtocols, removeStatus: http.StatusOK, output: "ok", stop: make(chan struct{}),
+	}
+	api.server = httptest.NewServer(http.HandlerFunc(api.serveHTTP))
+	address := api.server.Listener.Addr().String()
+	oldClient, oldDial, oldBase := podmanHTTPClient, podmanDialContext, podmanBaseURL
+	podmanDialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "tcp", address)
+	}
+	podmanHTTPClient = &http.Client{Transport: &http.Transport{DisableCompression: true, DialContext: podmanDialContext}}
+	podmanBaseURL = "http://podman"
+	t.Cleanup(func() {
+		api.stopOnce.Do(func() { close(api.stop) })
+		api.server.Close()
+		podmanHTTPClient, podmanDialContext, podmanBaseURL = oldClient, oldDial, oldBase
+	})
+	return api
+}
+
+func (api *fakeExecAPI) serveHTTP(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case r.URL.Path == libpodExecBasePath+"/version":
+		w.WriteHeader(api.preflightStatus)
+	case strings.Contains(r.URL.Path, "/containers/") && strings.HasSuffix(r.URL.Path, "/json"):
+		_ = json.NewEncoder(w).Encode(map[string]any{"Id": api.inspectContainerID, "State": map[string]any{"Running": api.running}, "Config": map[string]any{"Labels": map[string]string{"techulus.service.id": api.inspectServiceID, "techulus.deployment.id": api.inspectDeploymentID}}})
+	case strings.Contains(r.URL.Path, "/containers/") && strings.HasSuffix(r.URL.Path, "/exec"):
+		api.createCount++
+		var request struct {
+			Cmd []string `json:"Cmd"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&request)
+		if len(request.Cmd) == 3 {
+			api.createdCommand = request.Cmd[2]
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{"Id": "exec/id"})
+	case strings.HasSuffix(r.URL.Path, "/start"):
+		api.serveStart(w, r)
+	case strings.HasSuffix(r.URL.Path, "/json"):
+		_ = json.NewEncoder(w).Encode(map[string]any{"Running": false, "ExitCode": api.exitCode})
+	case strings.HasSuffix(r.URL.Path, "/remove"):
+		api.removeCount++
+		var request struct {
+			Force bool `json:"Force"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&request)
+		api.lastRemoveForce = request.Force
+		w.WriteHeader(api.removeStatus)
+		api.stopOnce.Do(func() { close(api.stop) })
+	default:
+		http.NotFound(w, r)
+	}
+}
+
+func (api *fakeExecAPI) serveStart(w http.ResponseWriter, r *http.Request) {
+	var request struct {
+		Detach bool `json:"Detach"`
+		TTY    bool `json:"Tty"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+		http.Error(w, "invalid start request", http.StatusBadRequest)
+		return
+	}
+	api.startRequestValid = !request.Detach && !request.TTY
+	if !api.startRequestValid {
+		http.Error(w, "invalid start options", http.StatusBadRequest)
+		return
+	}
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		panic("hijacking unsupported")
+	}
+	conn, buffer, err := hijacker.Hijack()
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+	if api.startStatus == http.StatusSwitchingProtocols {
+		_, _ = buffer.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n")
+	} else {
+		_, _ = buffer.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n")
+	}
+	_ = buffer.Flush()
+	if api.hang {
+		<-api.stop
+		return
+	}
+	middle := len(api.output) / 2
+	for stream, payload := range []string{api.output[:middle], api.output[middle:]} {
+		var header [8]byte
+		header[0] = byte(stream + 1)
+		binary.BigEndian.PutUint32(header[4:], uint32(len(payload)))
+		_, _ = conn.Write(header[:])
+		_, _ = conn.Write([]byte(payload))
+	}
+}
diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go
index e1212fb0..774997a2 100644
--- a/agent/internal/container/runtime.go
+++ b/agent/internal/container/runtime.go
@@ -12,7 +12,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"techulus/cloud-agent/internal/retry"
 	"techulus/cloud-agent/internal/wireguard"
@@ -59,46 +58,6 @@ func (b *limitedBuffer) Write(p []byte) (int, error) {
 	return n, nil
 }
 
-func ExecCommand(containerID, command string) (CommandResult, error) {
-	running, err := IsContainerRunning(containerID)
-	if err != nil {
-		return CommandResult{}, err
-	}
-	if !running {
-		return CommandResult{}, fmt.Errorf("container is not running")
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "podman", "exec", containerID, "/bin/sh", "-c", command)
-	var output limitedBuffer
-	cmd.Stdout, cmd.Stderr = &output, &output
-	err = cmd.Run()
-	outputText, truncated := output.snapshot()
-	outputText = strings.ToValidUTF8(outputText, "�")
-	if len(outputText) > CommandOutputLimit {
-		outputText = outputText[:CommandOutputLimit]
-		for !utf8.ValidString(outputText) {
-			outputText = outputText[:len(outputText)-1]
-		}
-		truncated = true
-	}
-	result := CommandResult{Output: outputText, Truncated: truncated}
-	if ctx.Err() == context.DeadlineExceeded {
-		result.TimedOut = true
-		result.ExitCode = 124
-		return result, nil
-	}
-	if err == nil {
-		return result, nil
-	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
-		result.ExitCode = exitErr.ExitCode()
-		return result, nil
-	}
-	return result, fmt.Errorf("failed to execute command: %w", err)
-}
-
 func ContainerExists(containerID string) (bool, error) {
 	cmd := exec.Command("podman", "inspect", "--format", "json", containerID)
 	output, err := cmd.CombinedOutput()
diff --git a/agent/internal/container/runtime_test.go b/agent/internal/container/runtime_test.go
index 7055c154..f983aba5 100644
--- a/agent/internal/container/runtime_test.go
+++ b/agent/internal/container/runtime_test.go
@@ -8,52 +8,8 @@ import (
 	"slices"
 	"strings"
 	"testing"
-	"time"
 )
 
-func TestExecCommand(t *testing.T) {
-	dir := t.TempDir()
-	script := `#!/bin/sh
-if [ "$1" = inspect ]; then echo '[{"State":{"Running":true}}]'; exit 0; fi
-case "$5" in
- success) printf 'hello';;
- failure) printf 'bad'; exit 7;;
- exact) head -c 65536 /dev/zero | tr '\0' x;;
- large) head -c 70000 /dev/zero | tr '\0' x;;
- timeout) sleep 1;;
-esac`
-	if err := os.WriteFile(filepath.Join(dir, "podman"), []byte(script), 0o755); err != nil {
-		t.Fatal(err)
-	}
-	t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
-	tests := []struct {
-		name                string
-		exit                int
-		truncated, timedOut bool
-	}{
-		{"success", 0, false, false}, {"failure", 7, false, false}, {"exact", 0, false, false}, {"large", 0, true, false}, {"timeout", 124, false, true},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			old := commandTimeout
-			if tt.timedOut {
-				commandTimeout = 10 * time.Millisecond
-			}
-			defer func() { commandTimeout = old }()
-			result, err := ExecCommand("container", tt.name)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if result.ExitCode != tt.exit || result.Truncated != tt.truncated || result.TimedOut != tt.timedOut {
-				t.Fatalf("unexpected result: %+v", result)
-			}
-			if len(result.Output) > CommandOutputLimit {
-				t.Fatalf("output exceeded limit: %d", len(result.Output))
-			}
-		})
-	}
-}
-
 func TestBuildPodmanPullArgs(t *testing.T) {
 	tests := []struct {
 		name      string
diff --git a/agent/internal/container/stats.go b/agent/internal/container/stats.go
index b5e1f795..a72dfd85 100644
--- a/agent/internal/container/stats.go
+++ b/agent/internal/container/stats.go
@@ -49,19 +49,19 @@ type podmanStatsReport struct {
 	Stats []podmanStatsSample `json:"Stats"`
 }
 
-const (
-	podmanSocketPath    = "/run/podman/podman.sock"
-	podmanStatsEndpoint = "http://podman/v4.0.0/libpod/containers/stats"
-)
+const podmanSocketPath = "/run/podman/podman.sock"
 
 var (
-	podmanStatsClient = &http.Client{Transport: &http.Transport{
+	podmanBaseURL     = "http://podman"
+	podmanDialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "unix", podmanSocketPath)
+	}
+	podmanHTTPClient = &http.Client{Transport: &http.Transport{
 		DisableCompression: true,
-		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
-			return (&net.Dialer{}).DialContext(ctx, "unix", podmanSocketPath)
+		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+			return podmanDialContext(ctx, network, address)
 		},
 	}}
-	podmanStatsURL = podmanStatsEndpoint
 )
 
 var previousResourceSamples = struct {
@@ -97,7 +97,7 @@ func CollectResourceStats() ([]ResourceStats, error) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	samples, err := fetchPodmanStats(ctx, podmanStatsClient, podmanStatsURL, containerIDs)
+	samples, err := fetchPodmanStats(ctx, podmanHTTPClient, podmanBaseURL+"/v4.0.0/libpod/containers/stats", containerIDs)
 	if err != nil {
 		return nil, err
 	}
diff --git a/agent/internal/container/stats_test.go b/agent/internal/container/stats_test.go
index bd7b65d6..a477d021 100644
--- a/agent/internal/container/stats_test.go
+++ b/agent/internal/container/stats_test.go
@@ -286,13 +286,13 @@ fi
 		w.WriteHeader(api.status)
 		_, _ = w.Write(api.body)
 	}))
-	previousClient := podmanStatsClient
-	previousURL := podmanStatsURL
-	podmanStatsClient = api.server.Client()
-	podmanStatsURL = api.server.URL
+	previousClient := podmanHTTPClient
+	previousBaseURL := podmanBaseURL
+	podmanHTTPClient = api.server.Client()
+	podmanBaseURL = api.server.URL
 	t.Cleanup(func() {
-		podmanStatsClient = previousClient
-		podmanStatsURL = previousURL
+		podmanHTTPClient = previousClient
+		podmanBaseURL = previousBaseURL
 		api.server.Close()
 	})
 	return api
diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
index b839de21..e8e7aeeb 100644
--- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
+++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/commands/page.tsx
@@ -227,7 +227,7 @@ export default function CommandsPage() {
 					/>
 					<div className="flex items-center justify-between gap-4">
 						<span className="text-xs text-muted-foreground">
-							{command.length}/4096 · 60 second timeout
+							{command.length}/4096 · 60 second foreground timeout
 						</span>
 						<Button
 							onClick={() => void runCommand()}
diff --git a/web/app/api/services/[id]/commands/route.ts b/web/app/api/services/[id]/commands/route.ts
index dca16557..96ea24f7 100644
--- a/web/app/api/services/[id]/commands/route.ts
+++ b/web/app/api/services/[id]/commands/route.ts
@@ -165,6 +165,7 @@ export async function POST(
 			"command",
 			{
 				commandRunId: commandId,
+				serviceId,
 				deploymentId: target.deploymentId,
 				containerId,
 				command: body.command as string,
diff --git a/web/lib/service-command-retention.ts b/web/lib/service-command-retention.ts
index ed38c560..9a8b60d3 100644
--- a/web/lib/service-command-retention.ts
+++ b/web/lib/service-command-retention.ts
@@ -6,13 +6,14 @@ import { DAY_IN_MILLISECONDS, subtractMilliseconds } from "@/lib/date";
 export const COMMAND_RETENTION_DAYS = 90;
 
 export async function cleanupOldServiceCommands(now = new Date()) {
-	return db
+	const result = await db
 		.delete(serviceCommands)
 		.where(
 			lt(
 				serviceCommands.createdAt,
 				subtractMilliseconds(now, COMMAND_RETENTION_DAYS * DAY_IN_MILLISECONDS),
 			),
-		)
-		.returning({ id: serviceCommands.id });
+		);
+
+	return result.rowCount ?? 0;
 }
diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts
index bcd71432..82b856cf 100644
--- a/web/lib/work-queue.ts
+++ b/web/lib/work-queue.ts
@@ -36,6 +36,7 @@ type ReconcileWorkPayload = {
 export type WorkPayloadByType = {
 	command: {
 		commandRunId: string;
+		serviceId: string;
 		deploymentId: string;
 		containerId: string;
 		command: string;
diff --git a/web/tests/service-command-retention.test.ts b/web/tests/service-command-retention.test.ts
index 218a284e..74826546 100644
--- a/web/tests/service-command-retention.test.ts
+++ b/web/tests/service-command-retention.test.ts
@@ -1,14 +1,12 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => {
-	const returning = vi.fn().mockResolvedValue([{ id: "command-1" }]);
-	const where = vi.fn(() => ({ returning }));
+	const where = vi.fn().mockResolvedValue({ rowCount: 3 });
 	const deleteCommand = vi.fn(() => ({ where }));
 	return {
 		db: { delete: deleteCommand },
 		deleteCommand,
 		where,
-		returning,
 		subtractMilliseconds: vi.fn(),
 	};
 });
@@ -29,7 +27,7 @@ describe("service command retention", () => {
 	beforeEach(() => {
 		mocks.deleteCommand.mockClear();
 		mocks.where.mockClear();
-		mocks.returning.mockClear();
+		mocks.where.mockResolvedValue({ rowCount: 3 });
 		mocks.subtractMilliseconds.mockReset();
 		mocks.subtractMilliseconds.mockReturnValue(
 			new Date("2026-05-07T00:00:00Z"),
@@ -47,6 +45,7 @@ describe("service command retention", () => {
 			90 * 86_400_000,
 		);
 		expect(mocks.deleteCommand).toHaveBeenCalledWith(serviceCommands);
-		expect(deleted).toEqual([{ id: "command-1" }]);
+		expect(deleted).toBe(3);
+		expect(mocks.where.mock.results[0]?.value).not.toHaveProperty("returning");
 	});
 });
diff --git a/web/tests/service-commands-route.test.ts b/web/tests/service-commands-route.test.ts
index 064cd05d..0727dc6f 100644
--- a/web/tests/service-commands-route.test.ts
+++ b/web/tests/service-commands-route.test.ts
@@ -116,6 +116,7 @@ describe("service commands route", () => {
 		const response = await post({
 			deploymentId: "deployment-1",
 			command: "printf hello",
+			serviceId: "attacker-controlled-service",
 		});
 		const body = (await response.json()) as { id: string; status: string };
 
@@ -136,6 +137,7 @@ describe("service commands route", () => {
 			"command",
 			{
 				commandRunId: body.id,
+				serviceId: "service-1",
 				deploymentId: "deployment-1",
 				containerId: "container-1",
 				command: "printf hello",

From a5e207a59d7c81e502ef9d15c45e0a42f5afcbfc Mon Sep 17 00:00:00 2001
From: Amp <amp@ampcode.com>
Date: Wed, 5 Aug 2026 12:25:20 +0000
Subject: [PATCH 6/6] Fix staticcheck error strings

Amp-Thread-ID: https://ampcode.com/threads/T-019fd17c-2d3f-74ce-ac88-8726a8936cdb
Co-authored-by: Arjun Komath <arjunkomath@gmail.com>
---
 agent/internal/container/libpod_exec.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/agent/internal/container/libpod_exec.go b/agent/internal/container/libpod_exec.go
index 44089ca5..5e119162 100644
--- a/agent/internal/container/libpod_exec.go
+++ b/agent/internal/container/libpod_exec.go
@@ -147,7 +147,7 @@ func execPreflight() error {
 	defer cancel()
 	resp, err := libpodRequest(ctx, http.MethodGet, libpodExecBasePath+"/version", nil)
 	if err != nil {
-		return fmt.Errorf("Podman 4.8 exec API preflight failed: %w", err)
+		return fmt.Errorf("podman 4.8 exec API preflight failed: %w", err)
 	}
 	defer resp.Body.Close()
 	if err := requireSuccess(resp, "Podman 4.8 exec API preflight"); err != nil {
@@ -235,7 +235,7 @@ func libpodJSON(method, path string, requestBody, responseBody any) error {
 			return err
 		}
 		if len(data) > libpodBodyLimit {
-			return fmt.Errorf("Podman response exceeds limit")
+			return fmt.Errorf("podman response exceeds limit")
 		}
 		return nil
 	}
@@ -245,7 +245,7 @@ func libpodJSON(method, path string, requestBody, responseBody any) error {
 		return err
 	}
 	if len(data) > libpodBodyLimit {
-		return fmt.Errorf("Podman response exceeds limit")
+		return fmt.Errorf("podman response exceeds limit")
 	}
 	if err := json.Unmarshal(data, responseBody); err != nil {
 		return fmt.Errorf("invalid Podman response: %w", err)
@@ -344,7 +344,7 @@ func copyMultiplexedOutput(reader io.Reader, output io.Writer) error {
 			if _, err := io.CopyN(io.Discard, reader, int64(remaining-messageLength)); err != nil {
 				return fmt.Errorf("invalid multiplexed error frame: %w", err)
 			}
-			return fmt.Errorf("Podman exec stream failed: %s", strings.TrimSpace(string(message)))
+			return fmt.Errorf("podman exec stream failed: %s", strings.TrimSpace(string(message)))
 		}
 		if header[0] != 0 && header[0] != 1 && header[0] != 2 {
 			return fmt.Errorf("invalid multiplexed stream %d", header[0])