-
- {zh ? "GPU / 端设备" : "GPU / edge devices"}
-
+
{zh ? "申请 GPU" : "GPU request"}
{requestText || (zh ? "未使用" : "Not used")}
@@ -1406,6 +1397,7 @@ function NodeWorkerTable({
{zh ? "实例名称" : "Worker name"}
Job
{zh ? "实例 IP" : "Worker IP"}
+
{zh ? "网络域" : "Domain"}
{zh ? "状态" : "Status"}
@@ -1418,6 +1410,19 @@ function NodeWorkerTable({
{worker.job}
{worker.ip}
+
+ {worker.domain ? (
+ <>
+
+ {worker.domain}
+ >
+ ) : (
+ "—"
+ )}
+
void;
+ onSuccess: (message: string) => void;
copy: Copy;
cloneJob?: Job | null;
editJob?: Job | null;
@@ -198,6 +205,8 @@ export function CreateJobModal({
>([]);
const [sshKeysLoaded, setSShKeysLoaded] = useState(false);
const [domains, setDomains] = useState<{ name: string; cidr: string }[]>([]);
+ const [reclaimableResources, setReclaimableResources] =
+ useState({});
const {
clusterDisplayNames,
nodes: allNodes,
@@ -231,6 +240,21 @@ export function CreateJobModal({
.catch(() => {});
}, []);
+ useEffect(() => {
+ if (!editJob || !restartAfterSave) return;
+ const selector = encodeURIComponent(`rlinf.io/job=${editJob.name}`);
+ fetch(`/api/v1/rlinf.io/v1alpha1/tasks?labelSelector=${selector}`)
+ .then((r) =>
+ r.ok ? r.json() : Promise.reject(new Error(`HTTP ${r.status}`)),
+ )
+ .then((data) =>
+ setReclaimableResources(
+ reclaimableResourcesForTasks((data.items ?? []) as CRDTask[]),
+ ),
+ )
+ .catch(() => setReclaimableResources({}));
+ }, [editJob, restartAfterSave]);
+
useEffect(() => {
fetch("/api/v1/ssh-user-keys")
.then((r) =>
@@ -373,6 +397,21 @@ export function CreateJobModal({
const [roleResources, setRoleResources] = useState<
Record
>(sourceJob ? cloneRR : defaultRoleResources);
+ const [placementModes, setPlacementModes] = useState<
+ Record
+ >(() =>
+ sourceJob
+ ? Object.fromEntries(
+ sourceJob.resources
+ .filter(
+ (resource) =>
+ Object.keys(parseNodeSelectorStr(resource.nodeSelector))
+ .length === 0,
+ )
+ .map((resource) => [resource.role, "model" as const]),
+ )
+ : {},
+ );
const [activeRoleTab, setActiveRoleTab] = useState(roles[0] ?? "");
useEffect(() => {
@@ -700,8 +739,13 @@ export function CreateJobModal({
const allocatable = Number(
node.status?.allocatable?.[resourceRequest.key] ?? 0,
);
- const used = Number(node.status?.used?.[resourceRequest.key] ?? 0);
- return Math.max(0, allocatable - used);
+ const used = node.status?.used?.[resourceRequest.key];
+ const nodeKey = `${node.metadata.namespace ?? ""}/${node.metadata.name}`;
+ return availableResource(
+ String(allocatable),
+ used,
+ reclaimableResources[nodeKey]?.[resourceRequest.key],
+ );
});
const available = freeByNode.reduce((sum, value) => sum + value, 0);
const requested = resource.replicas * resourceRequest.amount;
@@ -794,7 +838,19 @@ export function CreateJobModal({
const body = await resp.text();
throw new Error(`HTTP ${resp.status}: ${body}`);
}
- onClose();
+ onSuccess(
+ isEdit
+ ? restartAfterSave
+ ? zh
+ ? "任务已保存并提交重启"
+ : "Job saved and restart submitted"
+ : zh
+ ? "任务修改成功"
+ : "Job updated successfully"
+ : zh
+ ? "任务提交成功"
+ : "Job submitted successfully",
+ );
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
setErrorNonce((n) => n + 1);
@@ -1032,6 +1088,9 @@ export function CreateJobModal({
cluster={rr.cluster}
nodes={allNodes}
loading={nodesLoading}
+ reclaimableResources={reclaimableResources}
+ nodeSelector={rr.nodeSelector}
+ placementMode={placementModes[role]}
replicas={rr.replicas}
gpu={rr.gpu}
devices={rr.devices ?? []}
@@ -1041,6 +1100,12 @@ export function CreateJobModal({
[role]: { ...previous[role], ...placement },
}))
}
+ onPlacementModeChange={(placementMode) =>
+ setPlacementModes((previous) => ({
+ ...previous,
+ [role]: placementMode,
+ }))
+ }
/>
@@ -1449,20 +1514,34 @@ export function CreateJobModal({
)}
{step < 4 ? (
- goToStep(step + 1)}
- >
- {step === 2 &&
- roles.length > 1 &&
- (activeRoleTab || roles[0]) !== roles[roles.length - 1]
- ? zh
- ? "下一个角色"
- : "Next Role"
- : zh
- ? "下一步"
- : "Next"}
-
+ (() => {
+ const currentRole = activeRoleTab || roles[0];
+ const isLastRole = currentRole === roles[roles.length - 1];
+ const showNextRole =
+ step === 2 && roles.length > 1 && !isLastRole;
+ return (
+ {
+ if (showNextRole) {
+ const idx = roles.indexOf(currentRole);
+ setActiveRoleTab(roles[idx + 1]);
+ setError("");
+ } else {
+ goToStep(step + 1);
+ }
+ }}
+ >
+ {showNextRole
+ ? zh
+ ? "下一个角色"
+ : "Next Role"
+ : zh
+ ? "下一步"
+ : "Next"}
+
+ );
+ })()
) : (
task.phase)
- ).filter(Boolean);
- if (phases.length === 0 || job.phase !== "Running") return job.phase;
- if (phases.some((phase) => phase === "Failed")) return "Failed";
- if (phases.every((phase) => phase === "Succeeded")) return "Succeeded";
- if (!phases.some((phase) => phase === "Running")) return "Pending";
- return "Running";
-}
-
async function copyText(value: string) {
+ if (!value) return false;
+
try {
if (navigator.clipboard?.writeText) {
await navigator.clipboard.writeText(value);
@@ -170,6 +161,8 @@ export function JobsPage({
isMockMode,
selectedName,
onSelect,
+ onSelectNode,
+ onSelectCluster,
onCreate,
onClone,
onEditAndRestart,
@@ -179,6 +172,8 @@ export function JobsPage({
isMockMode: boolean;
selectedName: string;
onSelect: (name?: string) => void;
+ onSelectNode?: (name: string) => void;
+ onSelectCluster?: (id: string) => void;
onCreate?: () => void;
onClone?: (job: Job) => void;
onEditAndRestart?: (job: Job) => void;
@@ -190,11 +185,16 @@ export function JobsPage({
const [realJobs, setRealJobs] = useState([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState("");
+ const [actionNotice, setActionNotice] = useState("");
const [jobAction, setJobAction] = useState<
"start" | "stop" | "restart" | "delete" | null
>(null);
const [restartTarget, setRestartTarget] = useState(null);
const [deleteTarget, setDeleteTarget] = useState(null);
+ const [lifecycleConfirm, setLifecycleConfirm] = useState<{
+ job: Job;
+ action: "start" | "stop" | "clean-start" | "restart";
+ } | null>(null);
const [page, setPage] = useState(1);
const [pageSize, setPageSize] = useState(20);
const [sort, setSort] = useState<{
@@ -285,11 +285,23 @@ export function JobsPage({
setJobAction("delete");
setError("");
try {
+ const stopResp = await fetch(
+ `/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`,
+ {
+ method: "PATCH",
+ headers: { "Content-Type": "application/merge-patch+json" },
+ body: JSON.stringify({ spec: { stopped: true } }),
+ },
+ );
+ if (!stopResp.ok) throw new Error(`HTTP ${stopResp.status}`);
+ await waitForJobWorkersStopped(job);
+
const resp = await fetch(`/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`, {
method: "DELETE",
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
setRealJobs((prev) => prev.filter((j) => j.id !== job.id));
+ setActionNotice(zh ? "任务已删除" : "Job deleted");
return true;
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
@@ -299,19 +311,36 @@ export function JobsPage({
}
};
+ const waitForJobWorkersStopped = async (job: Job) => {
+ const deadline = Date.now() + 60_000;
+ const selector = encodeURIComponent(`rlinf.io/job=${job.name}`);
+ while (Date.now() < deadline) {
+ const [jobResp, tasksResp] = await Promise.all([
+ fetch(`/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`),
+ fetch(`/api/v1/rlinf.io/v1alpha1/tasks?labelSelector=${selector}`),
+ ]);
+ if (!jobResp.ok) throw new Error(`HTTP ${jobResp.status}`);
+ if (!tasksResp.ok) throw new Error(`HTTP ${tasksResp.status}`);
+ const current = crdToJob((await jobResp.json()) as CRDJob);
+ const tasks = (await tasksResp.json()) as {
+ items?: Array<{ status?: { phase?: string } }>;
+ };
+ const workersStopped = (tasks.items ?? []).every(
+ (task) => task.status?.phase === "Stopped",
+ );
+ if (current.phase === "Stopped" && workersStopped) {
+ return;
+ }
+ await new Promise((resolve) => window.setTimeout(resolve, 1000));
+ }
+ throw new Error(
+ zh
+ ? "等待 Worker 停止超时,任务未删除。"
+ : "Timed out waiting for workers to stop; the job was not deleted.",
+ );
+ };
+
const handleSetStopped = async (job: Job, stopped: boolean) => {
- if (
- !confirm(
- stopped
- ? zh
- ? `确定停止任务 "${job.name}" 吗?`
- : `Stop job "${job.name}"?`
- : zh
- ? `确定启动任务 "${job.name}" 吗?`
- : `Start job "${job.name}"?`,
- )
- )
- return false;
setJobAction(stopped ? "stop" : "start");
setError("");
try {
@@ -321,17 +350,27 @@ export function JobsPage({
body: JSON.stringify({ spec: { stopped } }),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+ if (stopped) await waitForJobWorkersStopped(job);
setRealJobs((prev) =>
prev.map((j) =>
j.id === job.id
? {
...j,
stopped,
- phase: stopped ? ("Stopped" as Phase) : ("Pending" as Phase),
+ phase: (stopped ? "Stopped" : "Pending") as Phase,
}
: j,
),
);
+ setActionNotice(
+ stopped
+ ? zh
+ ? "任务已停止,Worker 和 PVC 已清理"
+ : "Job stopped; workers and PVCs cleaned up"
+ : zh
+ ? "任务已提交启动"
+ : "Job start submitted",
+ );
return true;
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
@@ -363,6 +402,7 @@ export function JobsPage({
: item,
),
);
+ setActionNotice(zh ? "任务已提交重启" : "Job restart submitted");
return true;
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
@@ -372,6 +412,82 @@ export function JobsPage({
}
};
+ const waitForFailedJobCleanup = async (job: Job) => {
+ const deadline = Date.now() + 30_000;
+ while (Date.now() < deadline) {
+ const resp = await fetch(`/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`);
+ if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+ const current = crdToJob((await resp.json()) as CRDJob);
+ if (current.phase === "Stopped" && current.runningWorkers === 0) return;
+ await new Promise((resolve) => window.setTimeout(resolve, 1000));
+ }
+ throw new Error(
+ zh
+ ? "等待残留 Worker 清理超时,任务未启动。"
+ : "Timed out waiting for residual workers to stop; the job was not started.",
+ );
+ };
+
+ const handleCleanStart = async (job: Job) => {
+ setJobAction("restart");
+ setError("");
+ try {
+ const stopResp = await fetch(
+ `/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`,
+ {
+ method: "PATCH",
+ headers: { "Content-Type": "application/merge-patch+json" },
+ body: JSON.stringify({ spec: { stopped: true } }),
+ },
+ );
+ if (!stopResp.ok) throw new Error(`HTTP ${stopResp.status}`);
+ await waitForFailedJobCleanup(job);
+
+ const startResp = await fetch(
+ `/api/v1/rlinf.io/v1alpha1/jobs/${job.name}`,
+ {
+ method: "PATCH",
+ headers: { "Content-Type": "application/merge-patch+json" },
+ body: JSON.stringify({ spec: { stopped: false } }),
+ },
+ );
+ if (!startResp.ok) throw new Error(`HTTP ${startResp.status}`);
+ setRealJobs((prev) =>
+ prev.map((item) =>
+ item.id === job.id
+ ? { ...item, stopped: false, phase: "Pending" as Phase }
+ : item,
+ ),
+ );
+ setActionNotice(
+ zh ? "任务已清理并提交启动" : "Job cleaned and submitted",
+ );
+ return true;
+ } catch (e) {
+ setError(e instanceof Error ? e.message : String(e));
+ return false;
+ } finally {
+ setJobAction(null);
+ }
+ };
+
+ const confirmLifecycleAction = async () => {
+ if (!lifecycleConfirm) return;
+ const { job, action } = lifecycleConfirm;
+ const succeeded =
+ action === "stop"
+ ? await handleSetStopped(job, true)
+ : action === "start"
+ ? await handleSetStopped(job, false)
+ : action === "clean-start"
+ ? await handleCleanStart(job)
+ : await handleRestart(job);
+ if (succeeded) {
+ setLifecycleConfirm(null);
+ if (selectedName) onSelect(undefined);
+ }
+ };
+
const allJobs = realJobs;
const filtered = allJobs.filter((j) => {
const queryHit = `${j.id} ${j.displayName} ${j.type}`
@@ -409,6 +525,11 @@ export function JobsPage({
useEffect(() => {
if (page > totalPages) setPage(totalPages);
}, [page, totalPages]);
+ useEffect(() => {
+ if (!actionNotice) return;
+ const timer = window.setTimeout(() => setActionNotice(""), 4000);
+ return () => window.clearTimeout(timer);
+ }, [actionNotice]);
const selected =
selectedName && allJobs.length > 0
@@ -427,11 +548,18 @@ export function JobsPage({
lifecycleActions={{
pending: jobAction,
error,
- onStart: () => handleSetStopped(selected, false),
- onStop: () => handleSetStopped(selected, true),
+ onStart: () =>
+ setLifecycleConfirm({
+ job: selected,
+ action: selected.phase === "Failed" ? "clean-start" : "start",
+ }),
+ onStop: () =>
+ setLifecycleConfirm({ job: selected, action: "stop" }),
onRestart: () => setRestartTarget(selected),
onDelete: () => setDeleteTarget(selected),
}}
+ onSelectNode={onSelectNode}
+ onSelectCluster={onSelectCluster}
nodePullProgressMap={nodePullProgressMap}
nodeEventsMap={nodeEventsMap}
nodeDeviceModelMap={nodeDeviceModelMap}
@@ -444,7 +572,9 @@ export function JobsPage({
onRestart={() => {
const job = restartTarget;
setRestartTarget(null);
- void handleRestart(job);
+ void handleRestart(job).then((succeeded) => {
+ if (succeeded) onSelect(undefined);
+ });
}}
onEditRestart={
adminMode || !onEditAndRestart
@@ -472,12 +602,36 @@ export function JobsPage({
}}
/>
)}
+ {lifecycleConfirm && (
+ setLifecycleConfirm(null)}
+ onConfirm={confirmLifecycleAction}
+ />
+ )}
>
);
}
return (
+ {actionNotice && (
+
+
+ {actionNotice}
+ setActionNotice("")}
+ aria-label={zh ? "关闭提示" : "Dismiss notification"}
+ >
+ ×
+
+
+ )}
@@ -624,11 +778,19 @@ export function JobsPage({
job.phase === "Pending"
? aggregateJobEvents(job, nodeEventsMap)
: [];
+ const jobFailedMessage =
+ effectiveJobPhase(job) === "Failed"
+ ? job.taskStatuses
+ .filter((ts) => ts.phase === "Failed" && ts.message)
+ .map((ts) => ts.message)
+ .join("\n")
+ : undefined;
return (
28 ? " is-long" : ""}`}
+ title={job.id}
onClick={() => onSelect(job.id)}
>
{job.id}
@@ -643,11 +805,14 @@ export function JobsPage({
- {(jobPullProgress.length > 0 || jobEvents.length > 0) && (
+ {(jobPullProgress.length > 0 ||
+ jobEvents.length > 0 ||
+ jobFailedMessage) && (
)}
@@ -668,18 +833,12 @@ export function JobsPage({
handleSetStopped(job, true)}
- onRestart={() => {
- if (
- confirm(
- zh
- ? `确定重新启动任务 "${job.name}" 吗?`
- : `Restart job "${job.name}"?`,
- )
- ) {
- void handleRestart(job);
- }
- }}
+ onStop={() =>
+ setLifecycleConfirm({ job, action: "stop" })
+ }
+ onRestart={() =>
+ setLifecycleConfirm({ job, action: "restart" })
+ }
onDelete={() => setDeleteTarget(job)}
/>
) : (
@@ -689,7 +848,16 @@ export function JobsPage({
pending={jobAction !== null}
onClone={() => onClone?.(job)}
onDelete={() => setDeleteTarget(job)}
- onToggleStop={() => handleSetStopped(job, !job.stopped)}
+ onStart={() =>
+ setLifecycleConfirm({
+ job,
+ action:
+ job.phase === "Failed" ? "clean-start" : "start",
+ })
+ }
+ onStop={() =>
+ setLifecycleConfirm({ job, action: "stop" })
+ }
onRestart={() => setRestartTarget(job)}
/>
)}
@@ -741,6 +909,17 @@ export function JobsPage({
}}
/>
)}
+ {lifecycleConfirm && (
+ setLifecycleConfirm(null)}
+ onConfirm={confirmLifecycleAction}
+ />
+ )}
);
}
@@ -751,7 +930,8 @@ function JobActionMenu({
pending,
onClone,
onDelete,
- onToggleStop,
+ onStart,
+ onStop,
onRestart,
}: {
job: Job;
@@ -759,7 +939,8 @@ function JobActionMenu({
pending: boolean;
onClone: () => void;
onDelete: () => void;
- onToggleStop: () => void;
+ onStart: () => void;
+ onStop: () => void;
onRestart: () => void;
}) {
const [open, setOpen] = useState(false);
@@ -823,8 +1004,24 @@ function JobActionMenu({
};
}, [open]);
- const isStopped = job.stopped || job.phase === "Stopped";
- const isTerminal = ["Succeeded", "Failed"].includes(job.phase);
+ const isStartable =
+ job.stopped || job.phase === "Stopped" || job.phase === "Failed";
+ const isSucceeded = job.phase === "Succeeded";
+ const lifecycleLabel = isSucceeded
+ ? zh
+ ? "已成功完成的任务不能再次启动"
+ : "Succeeded jobs cannot be started again"
+ : job.phase === "Failed"
+ ? zh
+ ? "清理残留 Worker 后启动"
+ : "Clean residual workers, then start"
+ : isStartable
+ ? zh
+ ? "启动任务"
+ : "Start job"
+ : zh
+ ? "停止任务"
+ : "Stop job";
const handleToggle = () => {
setOpen((v) => !v);
@@ -832,68 +1029,41 @@ function JobActionMenu({
return (
-
- {isStopped ? : }
+
+
+ {zh ? "复制" : "Clone"}
+
+
+
+ {zh ? "重启" : "Restart"}
-
+ {isStartable ? : }
+ {isStartable ? (zh ? "启动" : "Start") : zh ? "停止" : "Stop"}
+
+
+
+
+
{open && (
<>
-
{
- setOpen(false);
- onClone();
- }}
- >
-
- {zh ? "复制" : "Clone"}
-
-
{
- setOpen(false);
- onRestart();
- }}
- >
-
- {zh ? "重启" : "Restart"}
-
{
@@ -960,6 +1130,140 @@ function AdminJobActions({
);
}
+function JobLifecycleConfirmDialog({
+ job,
+ action,
+ zh,
+ pending,
+ error,
+ onClose,
+ onConfirm,
+}: {
+ job: Job;
+ action: "start" | "stop" | "clean-start" | "restart";
+ zh: boolean;
+ pending: boolean;
+ error: string;
+ onClose: () => void;
+ onConfirm: () => Promise;
+}) {
+ useEffect(() => {
+ const closeOnEscape = (event: KeyboardEvent) => {
+ if (event.key === "Escape" && !pending) onClose();
+ };
+ document.addEventListener("keydown", closeOnEscape);
+ return () => document.removeEventListener("keydown", closeOnEscape);
+ }, [onClose, pending]);
+
+ const content = {
+ start: {
+ eyebrow: zh ? "任务启动" : "Start job",
+ title: zh ? "确认启动任务?" : "Start this job?",
+ description: zh
+ ? "任务将按当前配置重新进入调度队列。"
+ : "The job will re-enter the scheduling queue with its current configuration.",
+ confirm: zh ? "确认启动" : "Start job",
+ pending: zh ? "启动中…" : "Starting…",
+ icon: ,
+ },
+ stop: {
+ eyebrow: zh ? "任务停止" : "Stop job",
+ title: zh ? "确认停止任务?" : "Stop this job?",
+ description: zh
+ ? "平台将停止该任务的 Worker,当前运行连接会中断。"
+ : "The platform will stop this job's workers and interrupt active connections.",
+ confirm: zh ? "确认停止" : "Stop job",
+ pending: zh ? "停止中…" : "Stopping…",
+ icon: ,
+ },
+ "clean-start": {
+ eyebrow: zh ? "安全启动" : "Clean start",
+ title: zh ? "清理后启动任务?" : "Clean up and start?",
+ description: zh
+ ? "平台会先停止并清理失败任务残留的 Worker,确认清理完成后再启动。"
+ : "Residual workers will be stopped and cleaned up before the job starts.",
+ confirm: zh ? "清理后启动" : "Clean and start",
+ pending: zh ? "清理并启动中…" : "Cleaning and starting…",
+ icon: ,
+ },
+ restart: {
+ eyebrow: zh ? "任务重启" : "Restart job",
+ title: zh ? "确认重启任务?" : "Restart this job?",
+ description: zh
+ ? "任务将使用当前配置重新启动,现有运行连接会中断。"
+ : "The job will restart with its current configuration and interrupt active connections.",
+ confirm: zh ? "确认重启" : "Restart job",
+ pending: zh ? "重启中…" : "Restarting…",
+ icon: ,
+ },
+ }[action];
+
+ return (
+
+ event.target === event.currentTarget && !pending && onClose()
+ }
+ >
+
+
+
{content.icon}
+
+ {content.eyebrow}
+
{content.title}
+
+
+ ×
+
+
+
+
{content.description}
+
+ {zh ? "目标任务" : "Target job"}
+ {job.name}
+
+ {error && (
+
+ {error}
+
+ )}
+
+
+
+ {zh ? "取消" : "Cancel"}
+
+ void onConfirm()}
+ disabled={pending}
+ >
+ {pending ? (
+
+ ) : (
+ content.icon
+ )}
+ {pending ? content.pending : content.confirm}
+
+
+
+
+ );
+}
+
function DeleteJobDialog({
job,
zh,
@@ -1060,8 +1364,8 @@ function DeleteJobDialog({
)}
{pending
? zh
- ? "删除中…"
- : "Deleting…"
+ ? "正在停止并清理…"
+ : "Stopping and cleaning up…"
: zh
? "确认删除"
: "Delete job"}
@@ -1175,6 +1479,8 @@ export function JobDetailPage({
onBack,
onClone,
lifecycleActions,
+ onSelectNode,
+ onSelectCluster,
nodePullProgressMap = {},
nodeEventsMap = {},
nodeDeviceModelMap = {},
@@ -1185,6 +1491,8 @@ export function JobDetailPage({
onBack: () => void;
onClone?: () => void;
lifecycleActions: JobLifecycleActions;
+ onSelectNode?: (name: string) => void;
+ onSelectCluster?: (id: string) => void;
// Per-node pullProgress cache shared from JobsPage; used by the top
// StatusBadge hover to surface image pull progress while the job is Pending.
nodePullProgressMap?: Record;
@@ -1201,6 +1509,7 @@ export function JobDetailPage({
"workers",
);
const [taskNodes, setTaskNodes] = useState>({});
+ const [taskClusters, setTaskClusters] = useState>({});
const [tensorBoardProxy, setTensorBoardProxy] = useState("");
const [pullProgressMap, setPullProgressMap] = useState<
Record
@@ -1229,7 +1538,15 @@ export function JobDetailPage({
const [workerRoleFilter, setWorkerRoleFilter] = useState("All");
const [workerPage, setWorkerPage] = useState(1);
const [workerSort, setWorkerSort] = useState<{
- key: "name" | "role" | "node" | "kind" | "ip" | "createdAt" | "phase";
+ key:
+ | "name"
+ | "role"
+ | "cluster"
+ | "node"
+ | "kind"
+ | "ip"
+ | "createdAt"
+ | "phase";
direction: SortDirection;
}>({ key: "name", direction: "asc" });
const workerTableRef = useRef(null);
@@ -1266,6 +1583,7 @@ export function JobDetailPage({
const data = await resp.json();
const items = data.items ?? [];
const nodeMap: Record = {};
+ const clusterMap: Record = {};
const progressMap: Record = {};
const taskEventsMap: Record = {};
let tbProxy = "";
@@ -1273,6 +1591,7 @@ export function JobDetailPage({
const taskName = item.metadata?.name ?? "";
const observedNodes = item.status?.observedNodes ?? [];
nodeMap[taskName] = observedNodes.join(", ") || "—";
+ clusterMap[taskName] = item.metadata?.namespace ?? "—";
if (item.status?.tensorBoardProxy) {
tbProxy = item.status.tensorBoardProxy;
}
@@ -1291,6 +1610,7 @@ export function JobDetailPage({
}
}
setTaskNodes(nodeMap);
+ setTaskClusters(clusterMap);
setTensorBoardProxy(tbProxy);
setPullProgressMap(progressMap);
setTaskEventsMap(taskEventsMap);
@@ -1456,6 +1776,7 @@ export function JobDetailPage({
taskNodes[ts.name] ??
ts.observedNodes?.join(", ") ??
"—",
+ cluster: taskClusters[jobChildName] ?? taskClusters[ts.name] ?? "—",
phase: (ts.phase || "Pending") as Phase,
cpu: job.resources.find((item) => item.role === ts.name)?.cpu ?? "",
memory:
@@ -1467,6 +1788,7 @@ export function JobDetailPage({
`${ts.name}: worker state synced`,
`${ts.name}: waiting for runtime heartbeat`,
],
+ statusMessage: ts.message || undefined,
pullProgress:
pullProgressMap[jobChildName] ??
pullProgressMap[ts.name.toLowerCase()] ??
@@ -1512,6 +1834,7 @@ export function JobDetailPage({
jobId: job.id,
role,
node: pod.node || "—",
+ cluster: pod.taskNamespace || pod.namespace || "—",
phase: (pod.phase || "Pending") as Phase,
cpu: resource?.cpu ?? "",
memory: resource?.memory ?? "",
@@ -1522,6 +1845,7 @@ export function JobDetailPage({
`${role}: worker state synced`,
`${role}: waiting for runtime heartbeat`,
],
+ statusMessage: pod.message || undefined,
pullProgress: nodePullProgress,
events: workerEvents,
};
@@ -1567,6 +1891,8 @@ export function JobDetailPage({
return worker.name;
case "role":
return worker.role;
+ case "cluster":
+ return worker.cluster ?? "";
case "node":
return worker.node;
case "kind":
@@ -1667,6 +1993,13 @@ export function JobDetailPage({
{ id: "logs", label: c.common.logs },
{ id: "metrics", label: zh ? "监控" : "Metrics" },
];
+ const jobFailedMessage =
+ displayPhase === "Failed"
+ ? job.taskStatuses
+ .filter((ts) => ts.phase === "Failed" && ts.message)
+ .map((ts) => ts.message)
+ .join("\n")
+ : undefined;
return (
{tabs.map((tab) => (
@@ -1749,6 +2083,7 @@ export function JobDetailPage({
[
["name", zh ? "实例名称" : "Worker name"],
["role", zh ? "角色" : "Role"],
+ ["cluster", zh ? "集群" : "Cluster"],
["node", zh ? "节点" : "Node"],
["kind", zh ? "节点类型" : "Node type"],
["ip", zh ? "实例 IP" : "Worker IP"],
@@ -1782,6 +2117,12 @@ export function JobDetailPage({
worker.name === job.headerWorker
}
createdAt={formatWorkerCreatedAt(job.startedAt, index)}
+ onSelectNode={onSelectNode}
+ onSelectCluster={onSelectCluster}
+ podEventsMap={podEventsMap}
+ nodeEventsMap={nodeEventsMap}
+ nodePullProgressMap={nodePullProgressMap}
+ taskEventsMap={taskEventsMap}
/>
))}
@@ -1986,13 +2327,14 @@ function JobPublicOverview({
lifecycleActions,
jobPullProgress = [],
jobEvents = [],
+ jobFailedMessage,
}: {
job: Job;
copy: CopyType;
onBack: () => void;
runningWorkerCount: number;
totalWorkers: number;
- displayPhase: Phase;
+ displayPhase: JobDisplayPhase;
tensorBoardProxy?: string;
onClone?: () => void;
lifecycleActions: JobLifecycleActions;
@@ -2001,6 +2343,9 @@ function JobPublicOverview({
jobPullProgress?: PullProgressEntry[];
// 节点级 Warning 事件聚合,在 Pending 时与 pullProgress 一并展示。
jobEvents?: NodeEventEntry[];
+ // Failed 状态下聚合的异常原因(CrashLoopBackOff / ImagePullBackOff 等),
+ // 供状态徽标 "i" tooltip 展示。
+ jobFailedMessage?: string;
}) {
const zh = c.nav.overview === "总览";
const baseConfigRows = [
@@ -2029,11 +2374,14 @@ function JobPublicOverview({
- {(jobPullProgress.length > 0 || jobEvents.length > 0) && (
+ {(jobPullProgress.length > 0 ||
+ jobEvents.length > 0 ||
+ jobFailedMessage) && (
)}
@@ -2947,11 +3295,13 @@ function PullProgressInfo({
events = [],
zh,
emptyMessage,
+ statusMessage,
}: {
progress: PullProgressEntry[];
events?: NodeEventEntry[];
zh: boolean;
emptyMessage?: string;
+ statusMessage?: string;
}) {
const wrapperRef = useRef
(null);
const tooltipRef = useRef(null);
@@ -2962,6 +3312,13 @@ function PullProgressInfo({
above: boolean;
arrowLeft: number;
} | null>(null);
+ const recentEvents = [...events]
+ .sort((left, right) => {
+ const leftTime = Date.parse(left.lastTime ?? "") || 0;
+ const rightTime = Date.parse(right.lastTime ?? "") || 0;
+ return rightTime - leftTime;
+ })
+ .slice(0, 4);
const measure = () => {
const icon = wrapperRef.current;
@@ -3023,7 +3380,7 @@ function PullProgressInfo({
};
frame = window.requestAnimationFrame(track);
return () => window.cancelAnimationFrame(frame);
- }, [open, progress, events, emptyMessage]);
+ }, [open, progress, events, emptyMessage, statusMessage]);
const tooltipStyle: CSSProperties = pos
? {
@@ -3066,10 +3423,21 @@ function PullProgressInfo({
aria-hidden="true"
style={{ left: pos ? pos.arrowLeft - 6 : 0 }}
/>
- {progress.length === 0 && events.length === 0 && emptyMessage && (
+ {progress.length === 0 &&
+ events.length === 0 &&
+ !statusMessage &&
+ emptyMessage && (
+ <>
+ {zh ? "Worker 等待中" : "Worker pending"}
+ {emptyMessage}
+ >
+ )}
+ {statusMessage && (
<>
- {zh ? "Worker 等待中" : "Worker pending"}
- {emptyMessage}
+ {zh ? "异常原因" : "Failure Reason"}
+
+ {statusMessage}
+
>
)}
{progress.length > 0 && (
@@ -3107,12 +3475,19 @@ function PullProgressInfo({
})}
>
)}
- {events.length > 0 && (
+ {recentEvents.length > 0 && (
<>
{zh ? "Worker 事件" : "Worker Events"}
+ {events.length > recentEvents.length && (
+
+ {zh
+ ? `最近 ${recentEvents.length} 条`
+ : `Latest ${recentEvents.length}`}
+
+ )}
- {events.map((ev, i) => (
+ {recentEvents.map((ev, i) => (
;
isHeader?: boolean;
createdAt: string;
+ onSelectNode?: (name: string) => void;
+ onSelectCluster?: (id: string) => void;
+ podEventsMap: Record;
+ nodeEventsMap: Record;
+ nodePullProgressMap: Record;
+ taskEventsMap: Record;
}) {
const zh = c.nav.overview === "总览";
const [copied, setCopied] = useState(false);
@@ -3173,9 +3560,8 @@ function WorkerTableRow({
})
.catch(() => {});
}, []);
- const sshUser = sessionStorage.getItem("rlark-user-name") || "";
const sshJump = sshConfig?.sshJumpHost
- ? `${sshUser}@${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? ":" + sshConfig.sshJumpPort : ""}`
+ ? `${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? ":" + sshConfig.sshJumpPort : ""}`
: "";
const sshCommand = sshJump ? `ssh -J ${sshJump} root@${worker.name}` : "";
const handleCopy = async () => {
@@ -3192,8 +3578,10 @@ function WorkerTableRow({
-
- {worker.name}
+
+
+ {worker.name}
+
{isHeader
? "Header Worker"
@@ -3213,7 +3601,13 @@ function WorkerTableRow({
{worker.role}
- {worker.node}
+
+
+
+
{getNodeKindLabel(worker)}
@@ -3229,12 +3623,16 @@ function WorkerTableRow({
{worker.phase !== "Running" &&
(worker.phase === "Pending" ||
+ worker.phase === "Failed" ||
(worker.pullProgress && worker.pullProgress.length > 0) ||
(worker.events && worker.events.length > 0)) && (
{copied ? : }
@@ -3340,7 +3747,7 @@ function WorkerTableRow({
{expanded && (
-
+
@@ -3349,38 +3756,38 @@ function WorkerTableRow({
{worker.name}
-
-
- {sshCommand}
-
- {copied ? : }
-
-
+ {sshCommand && (
+
+
+ {sshCommand}
+
+ {copied ? : }
+
+
+ )}
+
+ {zh ? "集群" : "Cluster"}
+
+
{zh ? "角色" : "Role"}
{worker.role}
{zh ? "节点" : "Node"}
- {worker.node}
-
-
- {zh ? "申请 CPU" : "CPU request"}
-
- {worker.cpu || (zh ? "未申请" : "Not requested")}
-
-
-
- {zh ? "申请内存" : "Memory request"}
-
- {worker.memory || (zh ? "未申请" : "Not requested")}
-
+
{zh ? "申请 GPU" : "GPU request"}
@@ -3411,12 +3818,36 @@ function WorkerTableRow({
domainIPMap[
`${pod.namespace}/${pod.podNamespace}/${pod.podName}`
] ?? "";
+ const podPhase = (pod.phase || "Pending") as Phase;
+ // Mirror the worker-row tooltip aggregation so the pod
+ // subtable surfaces the same failure reason / image-pull
+ // progress / node events as the top-right status badge.
+ const podPullProgress =
+ podPhase === "Pending" && pod.node
+ ? (nodePullProgressMap[pod.node] ?? [])
+ : [];
+ const podEvents =
+ podPhase === "Pending"
+ ? (podEventsMap[pod.name] ?? []).length > 0
+ ? (podEventsMap[pod.name] ?? [])
+ : pod.node &&
+ (nodeEventsMap[pod.node] ?? []).length > 0
+ ? (nodeEventsMap[pod.node] ?? [])
+ : (taskEventsMap[
+ pod.taskName?.toLowerCase() ?? ""
+ ] ?? [])
+ : [];
return (
{pod.podName}
- {pod.node || "—"}
+
+
+
{pod.ip || "—"}
{pod.domain ? (
@@ -3440,10 +3871,36 @@ function WorkerTableRow({
)}
-
+
+
+ {podPhase !== "Running" &&
+ (podPhase === "Pending" ||
+ podPhase === "Failed" ||
+ podPullProgress.length > 0 ||
+ podEvents.length > 0) && (
+
+ )}
+
);
@@ -3463,3 +3920,55 @@ function WorkerTableRow({
>
);
}
+
+function WorkerClusterLink({
+ cluster,
+ onSelectCluster,
+}: {
+ cluster?: string;
+ onSelectCluster?: (id: string) => void;
+}) {
+ if (!onSelectCluster || !cluster || cluster === "—") {
+ return (
+
+ {cluster || "—"}
+
+ );
+ }
+ return (
+
onSelectCluster(cluster)}
+ title={cluster}
+ >
+ {cluster}
+
+ );
+}
+
+function WorkerNodeLink({
+ node,
+ onSelectNode,
+}: {
+ node: string;
+ onSelectNode?: (name: string) => void;
+}) {
+ if (!onSelectNode || !node || node === "—" || node.includes(",")) {
+ return (
+
+ {node || "—"}
+
+ );
+ }
+ return (
+
onSelectNode(node)}
+ title={node}
+ >
+ {node}
+
+ );
+}
diff --git a/apps/rlark-ui/src/pages/Login.tsx b/apps/rlark-ui/src/pages/Login.tsx
index 4d6231c..68eac5c 100644
--- a/apps/rlark-ui/src/pages/Login.tsx
+++ b/apps/rlark-ui/src/pages/Login.tsx
@@ -1,5 +1,5 @@
-import { useState, type FormEvent } from "react";
-import { ArrowRight, Shield } from "lucide-react";
+import { useEffect, useState, type FormEvent } from "react";
+import { AlertCircle, ArrowRight, Eye, EyeOff } from "lucide-react";
export function UserLogin({
onLogin,
@@ -8,9 +8,16 @@ export function UserLogin({
}) {
const [username, setUsername] = useState("user");
const [password, setPassword] = useState("");
+ const [showPassword, setShowPassword] = useState(false);
const [error, setError] = useState("");
const [loading, setLoading] = useState(false);
+ useEffect(() => {
+ if (!error) return;
+ const timer = window.setTimeout(() => setError(""), 4000);
+ return () => window.clearTimeout(timer);
+ }, [error]);
+
const handleSubmit = (e: FormEvent) => {
e.preventDefault();
if (!username.trim() || !password.trim()) {
@@ -46,50 +53,83 @@ export function UserLogin({
return (
+
+
);
diff --git a/apps/rlark-ui/src/pages/SSHKeys.tsx b/apps/rlark-ui/src/pages/SSHKeys.tsx
index 366ef21..c247f61 100644
--- a/apps/rlark-ui/src/pages/SSHKeys.tsx
+++ b/apps/rlark-ui/src/pages/SSHKeys.tsx
@@ -133,8 +133,8 @@ export function SSHKeysPage({ copy: c }: { copy: Copy }) {
.catch(() => {});
}, []);
const sshCommand = sshConfig?.sshJumpHost
- ? `ssh -J ${newUser}@${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? ":" + sshConfig.sshJumpPort : ""} root@
`
- : `ssh -J ${newUser}@:2222 root@`;
+ ? `ssh -J ${sshConfig.sshJumpHost}${sshConfig.sshJumpPort ? ":" + sshConfig.sshJumpPort : ""} root@`
+ : `ssh -J : root@`;
const handleCopy = () => {
navigator.clipboard.writeText(sshCommand);
@@ -259,7 +259,7 @@ export function SSHKeysPage({ copy: c }: { copy: Copy }) {
{zh ? "公钥信息" : "Key Info"}
- {zh ? "用户名" : "Username"}
+ {zh ? "公钥名称" : "Public Key Name"}
setNewUser(e.target.value)}
@@ -385,7 +385,7 @@ export function SSHKeysPage({ copy: c }: { copy: Copy }) {
toggleSort("user")}
diff --git a/apps/rlark-ui/src/pages/SystemConfig.tsx b/apps/rlark-ui/src/pages/SystemConfig.tsx
index 7f0e7ff..4e541f0 100644
--- a/apps/rlark-ui/src/pages/SystemConfig.tsx
+++ b/apps/rlark-ui/src/pages/SystemConfig.tsx
@@ -62,7 +62,7 @@ export function SystemConfigPage({ copy: c }: { copy: Copy }) {
};
const sshCommand = config.sshJumpHost
- ? `ssh -J @${config.sshJumpHost}${config.sshJumpPort ? ":" + config.sshJumpPort : ""} root@`
+ ? `ssh -J ${config.sshJumpHost}${config.sshJumpPort ? ":" + config.sshJumpPort : ""} root@`
: "";
const copySSHCommand = async () => {
diff --git a/apps/rlark-ui/src/styles.css b/apps/rlark-ui/src/styles.css
index cb654e9..568e976 100644
--- a/apps/rlark-ui/src/styles.css
+++ b/apps/rlark-ui/src/styles.css
@@ -1154,7 +1154,8 @@ button {
stroke-width: 2.5;
}
-.status-running .status-icon {
+.status-running .status-icon,
+.status-stopping .status-icon {
animation: status-spin 1.25s linear infinite;
}
@@ -1204,6 +1205,7 @@ button {
background: var(--orange);
}
+.status-stopping,
.status-stopped {
background: #f0f0f5;
color: #6b6b80;
@@ -1320,6 +1322,13 @@ button {
line-height: 1.55;
}
+.status-info-tooltip .status-message-entry {
+ color: #c0392b;
+ line-height: 1.55;
+ white-space: pre-wrap;
+ word-break: break-word;
+}
+
.status-info-tooltip .pull-entry code {
font-size: 10px;
color: #5f49aa;
@@ -1342,11 +1351,21 @@ button {
events are rendered. Mirrors .pull-entry's top border so the events
section reads as a sibling group. */
.status-info-tooltip .status-info-tooltip-section {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 8px;
border-top: 1px solid #edf0f5;
padding-top: 6px;
margin-top: 2px;
}
+.status-info-tooltip .status-info-tooltip-section small {
+ color: #8a95a6;
+ font-size: 9px;
+ font-weight: 500;
+}
+
/* First section heading (Image Pull Progress) has no preceding border. */
.status-info-tooltip > strong:first-child {
border-top: none;
@@ -1378,13 +1397,18 @@ button {
.status-info-tooltip .event-entry .event-object {
font-size: 10px;
color: #5f49aa;
- word-break: break-all;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
}
.status-info-tooltip .event-entry .event-message {
color: #667286;
+ display: -webkit-box;
font-size: 10px;
- word-break: break-word;
+ overflow: hidden;
+ -webkit-box-orient: vertical;
+ -webkit-line-clamp: 2;
}
.theme-dark .status-info {
@@ -1428,6 +1452,10 @@ button {
border-color: #2d394c;
}
+.theme-dark .status-info-tooltip .status-message-entry {
+ color: #ff6b6b;
+}
+
.progress-cell {
display: grid;
grid-template-columns: 1fr 32px;
@@ -2993,6 +3021,11 @@ tbody tr:hover {
font-size: 12px;
}
+.worker-name {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
.table-primary small {
display: block;
color: #98a1af;
@@ -3384,7 +3417,8 @@ tbody tr:hover {
.modal-backdrop {
position: fixed;
inset: 0;
- z-index: 20;
+ /* 必须高于 .action-dropdown(50),否则打开弹窗时下拉菜单会浮在遮罩之上 */
+ z-index: 60;
display: grid;
place-items: center;
background: rgba(16, 22, 32, 0.46);
@@ -4384,7 +4418,7 @@ tbody tr:hover {
}
.worker-table table {
- table-layout: auto;
+ table-layout: fixed;
width: 100%;
}
@@ -4392,6 +4426,24 @@ tbody tr:hover {
word-break: break-word;
}
+.worker-table th:first-child,
+.worker-table td:first-child {
+ width: 220px;
+}
+
+.worker-name-block {
+ min-width: 0;
+}
+
+.worker-name {
+ display: block;
+ min-width: 0;
+ max-width: 280px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
.worker-node-cell strong {
display: block;
color: var(--text);
@@ -5672,6 +5724,76 @@ tbody tr:hover {
z-index: 1200;
}
+.job-lifecycle-backdrop {
+ z-index: 1200;
+}
+
+.job-lifecycle-modal {
+ width: min(500px, calc(100vw - 32px));
+ border-radius: 18px;
+}
+
+.job-lifecycle-head {
+ display: grid;
+ grid-template-columns: 42px minmax(0, 1fr) 34px;
+ gap: 12px;
+ align-items: start;
+ padding: 22px 22px 18px;
+ border-bottom: 1px solid var(--line);
+}
+
+.job-lifecycle-icon {
+ display: grid;
+ place-items: center;
+ width: 42px;
+ height: 42px;
+ border-radius: 12px;
+ background: #eef8f4;
+ color: #16875d;
+}
+
+.job-lifecycle-modal.stop .job-lifecycle-icon {
+ background: #fff7e8;
+ color: #d97706;
+}
+
+.job-lifecycle-head h2 {
+ margin: 3px 0 0;
+ color: var(--text);
+ font-size: 20px;
+}
+
+.job-lifecycle-body {
+ padding: 18px 22px 16px;
+}
+
+.job-lifecycle-body > p {
+ margin: 0 0 14px;
+ color: var(--muted);
+ font-size: 12px;
+ line-height: 1.65;
+}
+
+.job-lifecycle-actions {
+ display: flex;
+ justify-content: flex-end;
+ gap: 9px;
+ padding: 14px 22px 20px;
+ border-top: 1px solid var(--line);
+}
+
+.job-lifecycle-actions button {
+ min-height: 36px;
+ padding: 0 15px;
+ border-radius: 10px;
+}
+
+.job-lifecycle-actions .primary-button {
+ display: inline-flex;
+ align-items: center;
+ gap: 7px;
+}
+
.delete-job-modal {
width: min(500px, calc(100vw - 32px));
border-radius: 18px;
@@ -5920,6 +6042,15 @@ tbody tr:hover {
background: #171b25;
}
+.theme-dark .job-lifecycle-modal {
+ background: #121a28;
+ border-color: #2a3547;
+}
+
+.theme-dark .job-lifecycle-icon {
+ background: rgba(22, 135, 93, 0.16);
+}
+
.theme-dark .delete-job-icon,
.theme-dark .delete-job-error {
background: rgba(248, 113, 113, 0.12);
@@ -6016,6 +6147,10 @@ tbody tr:hover {
.worker-console-table table {
min-width: 930px;
+ /* Auto layout lets each column size to its content so long cluster/node
+ names widen the table (scrollable via .worker-table-scroll) instead of
+ overflowing into neighbouring cells and overlapping their content. */
+ table-layout: auto;
}
.worker-console-table th,
@@ -6031,9 +6166,83 @@ tbody tr:hover {
text-transform: uppercase;
}
-.table-node-name {
+/* 集群/节点 chip:与 .role-chip / .node-kind-chip 同一风格体系,
+ 可点击变体通过 hover 淡紫底提示可跳转。 */
+.worker-chip {
+ display: inline-flex;
+ align-items: center;
+ max-width: 260px;
+ border-radius: 999px;
+ background: #f0f3f7;
+ color: #667182;
+ padding: 6px 9px;
+ font-size: 11px;
+ font-weight: 800;
+ white-space: nowrap;
+}
+
+.worker-chip-link {
+ border: 0;
+ cursor: pointer;
+ transition:
+ background 0.12s ease,
+ color 0.12s ease;
+}
+
+.worker-chip-link:hover {
+ background: rgba(124, 58, 237, 0.08);
+ color: var(--blue);
+}
+
+.worker-link-label {
+ min-width: 0;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.theme-dark .worker-chip {
+ background: #202b3d;
+ color: #b1bdd0;
+}
+
+.theme-dark .worker-chip-link:hover {
+ background: rgba(167, 139, 250, 0.15);
+ color: #c4b5fd;
+}
+
+.job-action-notice {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ margin-bottom: 14px;
+ border: 1px solid color-mix(in srgb, #23a36d 35%, var(--line));
+ border-radius: 10px;
+ background: color-mix(in srgb, #23a36d 8%, var(--surface));
color: var(--text);
- font-size: 12px;
+ padding: 10px 12px;
+ font-size: 13px;
+ font-weight: 650;
+}
+
+.job-action-notice > svg {
+ color: #1c9b64;
+}
+
+.job-action-notice > span {
+ flex: 1;
+}
+
+.job-action-notice > button {
+ border: 0;
+ background: transparent;
+ color: var(--muted);
+ font-size: 18px;
+ cursor: pointer;
+}
+
+.app-job-submit-notice {
+ margin: 18px 24px 0;
}
.node-kind-chip {
@@ -8416,19 +8625,16 @@ tbody tr:hover {
}
.placement-node-head {
- display: flex;
- align-items: center;
- justify-content: space-between;
- gap: 8px;
min-width: 0;
- padding-right: 22px;
}
.placement-node-head strong {
- overflow: hidden;
+ display: block;
+ width: 100%;
font-size: 14px;
- text-overflow: ellipsis;
- white-space: nowrap;
+ line-height: 1.3;
+ overflow-wrap: anywhere;
+ word-break: break-word;
}
.placement-node-head em {
@@ -8448,12 +8654,30 @@ tbody tr:hover {
}
.placement-node-location {
+ flex: 1;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
+.placement-node-meta,
+.placement-node-state {
+ display: flex;
+ align-items: center;
+}
+
+.placement-node-meta {
+ justify-content: space-between;
+ gap: 8px;
+}
+
+.placement-node-state {
+ flex: none;
+ justify-content: flex-end;
+ gap: 6px;
+}
+
.placement-node-specs {
display: flex;
align-items: center;
@@ -8519,16 +8743,10 @@ tbody tr:hover {
color: #d64d68;
font-weight: 700;
}
-.placement-node-alert {
- position: absolute;
- right: 12px;
- bottom: 12px;
+.placement-node-state > svg {
color: #d64d68;
}
.placement-node-check {
- position: absolute;
- top: 10px;
- right: 10px;
justify-content: center;
width: 18px;
height: 18px;
@@ -9146,6 +9364,28 @@ tbody tr:hover {
font-size: 13px;
}
+.jobs-table-panel th:first-child,
+.jobs-table-panel td:first-child {
+ width: 170px;
+ min-width: 170px;
+}
+
+.jobs-table-panel .job-id-cell {
+ width: 100%;
+ min-width: 0;
+}
+
+.jobs-table-panel .job-id-cell strong {
+ overflow-wrap: anywhere;
+ word-break: break-word;
+ line-height: 1.35;
+}
+
+.jobs-table-panel .job-id-cell.is-long strong {
+ font-size: 11px;
+ line-height: 1.3;
+}
+
.jobs-table-panel .link-cell small {
display: block;
color: #98a1af;
@@ -10145,9 +10385,40 @@ tbody tr:hover {
white-space: nowrap;
}
-.job-quick-lifecycle {
- width: 34px;
- height: 34px;
+.jobs-list-page .action-tooltip::after {
+ right: 50%;
+ transform: translate(50%, 3px);
+}
+
+.jobs-list-page .action-tooltip:hover::after,
+.jobs-list-page .action-tooltip:focus-within::after {
+ transform: translate(50%, 0);
+}
+
+.job-row-action {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ gap: 5px;
+ min-height: 32px;
+ padding: 0 9px;
+ border: 1px solid var(--line);
+ border-radius: 8px;
+ background: var(--surface);
+ color: var(--text);
+ font-size: 12px;
+ font-weight: 600;
+ cursor: pointer;
+}
+
+.job-row-action:hover:not(:disabled) {
+ border-color: rgba(124, 77, 255, 0.35);
+ color: var(--purple);
+}
+
+.job-row-action:disabled {
+ opacity: 0.5;
+ cursor: not-allowed;
}
.job-quick-lifecycle.start {
@@ -10992,8 +11263,8 @@ input[type="radio"]:disabled {
font-weight: 700;
}
-.node-worker-table th:last-child,
-.node-worker-table td:last-child {
+.node-worker-table > thead > tr > th:last-child,
+.node-worker-table > tbody > tr > td:last-child {
position: sticky;
right: 0;
z-index: 1;
@@ -11001,11 +11272,11 @@ input[type="radio"]:disabled {
box-shadow: -8px 0 12px -12px rgba(15, 23, 42, 0.5);
}
-.node-worker-table th:last-child {
+.node-worker-table > thead > tr > th:last-child {
z-index: 2;
}
-.node-worker-table tbody tr:last-child td {
+.node-worker-table > tbody > tr:last-child > td {
border-bottom: 0;
}
@@ -11369,27 +11640,82 @@ input[type="radio"]:disabled {
gap: 9px;
}
-.admin-label-editor .label-edit-row {
- grid-template-columns: minmax(180px, 0.8fr) minmax(180px, 1.2fr) 36px;
- padding: 8px;
+.admin-node-managed-fields {
+ display: grid;
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ gap: 10px;
+ padding: 14px;
border: 1px solid var(--line);
- border-radius: 11px;
+ border-radius: 12px;
background: var(--canvas);
}
-.admin-label-editor .label-edit-row code {
- overflow: hidden;
- color: var(--blue);
- text-overflow: ellipsis;
- white-space: nowrap;
+.admin-node-managed-field {
+ display: grid;
+ gap: 7px;
}
-.admin-label-editor .label-edit-row input,
-.admin-label-editor .label-add-row input {
- height: 36px;
- border: 1px solid var(--line-strong);
- border-radius: 9px;
- background: var(--panel);
+.admin-node-managed-field label {
+ color: var(--soft);
+ font-size: 11px;
+ font-weight: 700;
+}
+
+.admin-node-managed-field input {
+ width: 100%;
+ height: 38px;
+ padding: 0 11px;
+ border: 1px solid var(--line-strong);
+ border-radius: 9px;
+ background: var(--panel);
+ color: var(--ink);
+ box-sizing: border-box;
+}
+
+.admin-node-managed-categories {
+ grid-column: 1 / -1;
+}
+
+.admin-node-extension-head {
+ margin-top: 8px;
+}
+
+.admin-node-extension-head strong,
+.admin-node-extension-head small {
+ display: block;
+}
+
+.admin-node-extension-head strong {
+ font-size: 12px;
+}
+
+.admin-node-extension-head small {
+ margin-top: 3px;
+ color: var(--muted);
+ font-size: 10px;
+}
+
+.admin-label-editor .label-edit-row {
+ grid-template-columns: minmax(180px, 0.8fr) minmax(180px, 1.2fr) 36px;
+ padding: 8px;
+ border: 1px solid var(--line);
+ border-radius: 11px;
+ background: var(--canvas);
+}
+
+.admin-label-editor .label-edit-row code {
+ overflow: hidden;
+ color: var(--blue);
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.admin-label-editor .label-edit-row input,
+.admin-label-editor .label-add-row input {
+ height: 36px;
+ border: 1px solid var(--line-strong);
+ border-radius: 9px;
+ background: var(--panel);
color: var(--ink);
outline: none;
}
@@ -11463,6 +11789,10 @@ input[type="radio"]:disabled {
.admin-label-editor .label-add-row {
grid-template-columns: 1fr;
}
+
+ .admin-node-managed-fields {
+ grid-template-columns: 1fr;
+ }
}
.sort-th {
@@ -12912,6 +13242,7 @@ tr.clickable:hover {
.node-resource-table-panel {
padding: 0;
overflow: hidden;
+ border-radius: 14px;
}
.node-resource-table-summary {
@@ -12964,33 +13295,53 @@ tr.clickable:hover {
}
.node-resource-table-head {
- padding: 9px 16px;
+ padding: 10px 16px;
border-bottom: 1px solid var(--line);
background: var(--canvas);
color: var(--muted);
- font-size: 10px;
+ font-size: 9px;
font-weight: 700;
letter-spacing: 0.04em;
}
.node-resource-row {
width: 100%;
- padding: 12px 16px;
+ min-height: 58px;
+ padding: 9px 16px;
border: 0;
border-bottom: 1px solid var(--line);
background: transparent;
color: var(--ink);
text-align: left;
- cursor: pointer;
+ cursor: default;
transition: background 0.15s ease;
}
+.node-detail-link {
+ min-width: 0;
+ padding: 5px 7px;
+ margin: -5px -7px;
+ border: 0;
+ border-radius: 7px;
+ background: transparent;
+ color: inherit;
+ text-align: left;
+ cursor: pointer;
+}
+
+.node-detail-link:hover,
+.node-detail-link:focus-visible {
+ background: rgba(124, 58, 237, 0.09);
+ color: var(--blue);
+ outline: none;
+}
+
.node-resource-row:last-child {
border-bottom: 0;
}
.node-resource-row:hover {
- background: var(--skyblue-bg);
+ background: color-mix(in srgb, var(--skyblue-bg) 72%, var(--panel));
}
.node-resource-table-head.has-admin-actions,
@@ -13130,6 +13481,112 @@ tr.clickable:hover {
justify-content: flex-end;
}
+.admin-node-management-page {
+ display: flex;
+ flex-direction: column;
+ gap: 14px;
+ padding-top: 18px;
+}
+
+.admin-node-management-page > .section-heading {
+ margin-bottom: 0;
+}
+
+.admin-node-page-heading {
+ align-items: flex-end;
+ gap: 24px;
+ min-height: 74px;
+ padding: 4px 0 16px;
+ border-bottom: 1px solid var(--line);
+}
+
+.admin-node-page-heading h2 {
+ margin: 5px 0 6px;
+ font-size: 24px;
+ line-height: 1.2;
+}
+
+.admin-node-page-heading p {
+ max-width: 620px;
+ margin: 0;
+ color: var(--muted);
+ font-size: 12px;
+ line-height: 1.6;
+}
+
+.admin-node-page-heading .admin-node-list-actions {
+ flex: 0 0 auto;
+ max-width: 540px;
+ padding-bottom: 1px;
+}
+
+.admin-node-page-heading .admin-node-list-actions button {
+ min-height: 34px;
+}
+
+.admin-node-overview {
+ display: grid;
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ overflow: hidden;
+ border: 1px solid var(--line);
+ border-radius: 14px;
+ background: var(--panel);
+ box-shadow: var(--shadow-soft);
+}
+
+.admin-node-overview > div {
+ min-width: 0;
+ display: grid;
+ grid-template-columns: 34px minmax(0, 1fr) auto;
+ grid-template-rows: auto auto;
+ align-items: center;
+ column-gap: 10px;
+ padding: 13px 16px;
+ border-right: 1px solid var(--line);
+}
+
+.admin-node-overview > div:last-child {
+ border-right: 0;
+}
+
+.admin-node-overview-icon {
+ grid-row: 1 / 3;
+ width: 34px;
+ height: 34px;
+ display: grid;
+ place-items: center;
+ border-radius: 10px;
+ color: #495de6;
+ background: rgba(73, 93, 230, 0.1);
+}
+
+.admin-node-overview-icon.online,
+.admin-node-overview-icon.ready {
+ color: #14875e;
+ background: rgba(20, 135, 94, 0.1);
+}
+
+.admin-node-overview-icon.clusters {
+ color: #7c3aed;
+ background: rgba(124, 58, 237, 0.1);
+}
+
+.admin-node-overview small {
+ align-self: end;
+ overflow: hidden;
+ color: var(--muted);
+ font-size: 10px;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.admin-node-overview strong {
+ grid-column: 3;
+ grid-row: 1 / 3;
+ font-size: 22px;
+ line-height: 1;
+}
+
.admin-node-batch-panel {
margin-bottom: 16px;
padding: 16px;
@@ -15874,46 +16331,62 @@ tr.clickable:hover {
/* ── Admin Login ── */
.admin-login-page {
min-height: 100vh;
- background: var(--canvas);
+ position: relative;
+ overflow: hidden;
+ background:
+ linear-gradient(rgba(255, 255, 255, 0.76), rgba(247, 249, 255, 0.88)),
+ radial-gradient(circle at 18% 15%, #dbeafe 0, transparent 34%),
+ radial-gradient(circle at 82% 82%, #ede9fe 0, transparent 32%), #f5f7fb;
display: flex;
flex-direction: column;
}
-.admin-login-topbar {
- display: flex;
- align-items: center;
- justify-content: space-between;
- padding: 0 24px;
- height: 56px;
- border-bottom: 1px solid var(--line);
- background: var(--panel);
- flex-shrink: 0;
+.admin-login-page::before {
+ content: "";
+ position: absolute;
+ inset: 0;
+ opacity: 0.32;
+ background-image:
+ linear-gradient(rgba(90, 105, 160, 0.08) 1px, transparent 1px),
+ linear-gradient(90deg, rgba(90, 105, 160, 0.08) 1px, transparent 1px);
+ background-size: 48px 48px;
+ mask-image: linear-gradient(to bottom, black, transparent 78%);
+ pointer-events: none;
}
-.admin-login-topbar .admin-brand {
- display: flex;
- align-items: center;
- gap: 12px;
+.user-login-orb {
+ position: absolute;
+ border-radius: 999px;
+ filter: blur(2px);
+ pointer-events: none;
}
-.admin-login-topbar .admin-brand .brand-logo {
- width: 104px;
- height: 34px;
- flex-shrink: 0;
+.user-login-orb-one {
+ width: 360px;
+ height: 360px;
+ top: -180px;
+ right: -100px;
+ background: rgba(99, 102, 241, 0.13);
}
-.admin-login-topbar .admin-brand-text {
- display: flex;
- align-items: center;
- line-height: 1;
+.user-login-orb-two {
+ width: 280px;
+ height: 280px;
+ bottom: -150px;
+ left: -80px;
+ background: rgba(59, 130, 246, 0.12);
}
-.admin-login-topbar .admin-brand-text span {
- font-size: 13px;
- letter-spacing: 0.02em;
- color: var(--muted);
- font-weight: 700;
- text-transform: uppercase;
+.login-inline-error {
+ width: 100%;
+ min-height: 20px;
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ margin: -8px 0 2px;
+ color: #dc2626;
+ font-size: 11px;
+ line-height: 1.4;
}
.admin-login-body {
@@ -15922,45 +16395,94 @@ tr.clickable:hover {
align-items: center;
justify-content: center;
padding: 40px 20px;
+ position: relative;
+ z-index: 1;
+}
+
+.admin-login-panel {
+ position: relative;
+ width: 420px;
+ max-width: 100%;
}
.admin-login-card {
- width: 380px;
+ width: 100%;
max-width: 100%;
background: var(--panel);
- border: 1px solid var(--line);
- border-radius: 16px;
- padding: 40px 32px 32px;
+ border: 1px solid rgba(255, 255, 255, 0.9);
+ border-radius: 24px;
+ padding: 38px 38px 30px;
display: flex;
flex-direction: column;
align-items: center;
- gap: 6px;
- box-shadow: 0 4px 24px rgba(0, 0, 0, 0.06);
+ gap: 8px;
+ box-shadow:
+ 0 28px 70px rgba(45, 55, 100, 0.14),
+ 0 4px 14px rgba(45, 55, 100, 0.06);
+ backdrop-filter: blur(18px);
}
.admin-login-card h2 {
- margin: 8px 0 0;
- font-size: 20px;
- font-weight: 600;
+ margin: 5px 0 4px;
+ font-size: 26px;
+ font-weight: 700;
+ letter-spacing: -0.02em;
color: var(--ink);
}
.admin-login-card .muted {
font-size: 13px;
color: var(--muted);
- margin: 0 0 20px;
+ margin: 0;
text-align: center;
}
-.admin-login-logo {
- width: 56px;
- height: 56px;
- border-radius: 14px;
- background: var(--hover);
+.user-login-brand-logo {
+ width: 190px;
+ max-width: 100%;
+ height: auto;
+ object-fit: contain;
+}
+
+.user-login-brand {
+ width: 100%;
display: flex;
align-items: center;
justify-content: center;
- color: var(--blue);
+ padding-bottom: 17px;
+ margin-bottom: 10px;
+ border-bottom: 1px solid var(--line);
+}
+
+.user-login-heading {
+ width: 100%;
+ margin-bottom: 18px;
+ text-align: center;
+}
+
+.user-login-heading .muted {
+ text-align: center;
+}
+
+.admin-login-card-brand {
+ position: relative;
+}
+
+.admin-login-card-brand .user-login-brand-logo {
+ width: 176px;
+}
+
+.admin-login-badge {
+ position: absolute;
+ right: 0;
+ bottom: 16px;
+ padding: 4px 7px;
+ border: 1px solid rgba(99, 102, 241, 0.2);
+ border-radius: 6px;
+ color: #5b55d6;
+ background: rgba(99, 102, 241, 0.08);
+ font-size: 9px;
+ font-weight: 800;
}
.admin-login-field {
@@ -15973,38 +16495,97 @@ tr.clickable:hover {
.admin-login-field label {
font-size: 12px;
- font-weight: 500;
+ font-weight: 650;
color: var(--soft);
}
.admin-login-field input {
width: 100%;
- height: 40px;
- padding: 0 14px;
+ height: 46px;
+ padding: 0 15px;
border: 1px solid var(--line);
- border-radius: 8px;
+ border-radius: 10px;
font-size: 14px;
color: var(--ink);
- background: var(--canvas);
+ background: #f8f9fd;
outline: none;
- transition: border-color 0.15s;
+ transition:
+ border-color 0.15s,
+ box-shadow 0.15s,
+ background 0.15s;
box-sizing: border-box;
}
.admin-login-field input:focus {
border-color: var(--blue);
+ background: #fff;
+ box-shadow: 0 0 0 3px rgba(109, 93, 252, 0.11);
+}
+
+.admin-login-field input::placeholder {
+ color: var(--muted);
+ opacity: 0.48;
+ font-weight: 400;
+}
+
+.admin-login-password {
+ position: relative;
+ width: 100%;
+}
+
+.admin-login-password input {
+ padding-right: 44px;
+}
+
+.admin-login-password-toggle {
+ position: absolute;
+ top: 50%;
+ right: 8px;
+ width: 32px;
+ height: 32px;
+ padding: 0;
+ transform: translateY(-50%);
+ border: 0;
+ border-radius: 6px;
+ background: transparent;
+ color: var(--muted);
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ cursor: pointer;
+}
+
+.admin-login-password-toggle:hover,
+.admin-login-password-toggle:focus-visible {
+ color: var(--blue);
+ background: var(--hover);
+ outline: none;
}
.admin-login-btn {
width: 100%;
- height: 42px;
- margin-top: 8px;
+ height: 48px;
+ margin-top: 10px;
font-size: 14px;
font-weight: 600;
+ justify-content: center;
+ gap: 9px;
+ border: 0;
+ border-radius: 11px;
+ background: linear-gradient(135deg, #635bff, #7c3aed);
+ box-shadow: 0 12px 24px rgba(109, 72, 240, 0.24);
+ transition:
+ transform 0.15s,
+ box-shadow 0.15s;
+}
+
+.admin-login-btn:not(:disabled):hover {
+ transform: translateY(-1px);
+ box-shadow: 0 15px 28px rgba(109, 72, 240, 0.3);
}
.admin-login-back {
- margin-top: 20px;
+ margin-top: 17px;
font-size: 12px;
color: var(--muted);
text-decoration: none;
@@ -16026,10 +16607,27 @@ tr.clickable:hover {
box-shadow: 0 4px 24px rgba(0, 0, 0, 0.3);
}
+.theme-dark .login-inline-error {
+ color: #fca5a5;
+}
+
+.theme-dark .admin-login-badge {
+ border-color: rgba(196, 181, 253, 0.24);
+ color: #c4b5fd;
+ background: rgba(167, 139, 250, 0.12);
+}
+
.theme-dark .admin-login-field input {
background: var(--canvas);
}
+@media (max-width: 520px) {
+ .admin-login-card {
+ padding: 30px 24px 26px;
+ border-radius: 20px;
+ }
+}
+
/* ── DAG Editor ── */
.dag-toolbar {
display: flex;
@@ -16892,3 +17490,53 @@ tr.clickable:hover {
width: fit-content;
}
}
+
+@media (max-width: 1080px) {
+ .admin-node-page-heading {
+ align-items: flex-start;
+ flex-direction: column;
+ gap: 14px;
+ }
+
+ .admin-node-page-heading .admin-node-list-actions {
+ max-width: none;
+ justify-content: flex-start;
+ }
+
+ .admin-node-overview {
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ }
+
+ .admin-node-overview > div:nth-child(2) {
+ border-right: 0;
+ }
+
+ .admin-node-overview > div:nth-child(-n + 2) {
+ border-bottom: 1px solid var(--line);
+ }
+}
+
+@media (max-width: 640px) {
+ .admin-node-page-heading {
+ min-height: auto;
+ padding: 2px 0 14px;
+ }
+
+ .admin-node-page-heading .admin-node-list-actions {
+ width: 100%;
+ }
+
+ .admin-node-page-heading .admin-node-list-actions button {
+ flex: 1 1 calc(50% - 8px);
+ }
+
+ .admin-node-overview > div {
+ grid-template-columns: 30px minmax(0, 1fr) auto;
+ padding: 11px;
+ }
+
+ .admin-node-overview-icon {
+ width: 30px;
+ height: 30px;
+ }
+}
diff --git a/apps/rlark-ui/src/types.ts b/apps/rlark-ui/src/types.ts
index 6c90aff..5548dba 100644
--- a/apps/rlark-ui/src/types.ts
+++ b/apps/rlark-ui/src/types.ts
@@ -84,6 +84,21 @@ export interface CRDJobTask {
};
}
+export interface CRDTask {
+ metadata: {
+ name: string;
+ namespace?: string;
+ };
+ spec?: {
+ kubernetes?: {
+ workload?: CRDWorkload;
+ };
+ };
+ status?: {
+ observedNodes?: string[];
+ };
+}
+
export interface CRDJob {
apiVersion: string;
kind: string;
diff --git a/apps/rlark-ui/src/utils/jobPhase.ts b/apps/rlark-ui/src/utils/jobPhase.ts
new file mode 100644
index 0000000..3e725f0
--- /dev/null
+++ b/apps/rlark-ui/src/utils/jobPhase.ts
@@ -0,0 +1,26 @@
+import type { Job, Phase } from "../data";
+
+export type JobDisplayPhase = Phase | "Stopping";
+
+export function effectiveJobPhase(
+ job: Job,
+ workerPhases?: string[],
+): JobDisplayPhase {
+ const phases = (
+ workerPhases ?? job.taskStatuses.map((task) => task.phase)
+ ).filter(Boolean);
+ if (job.stopped) {
+ if (job.phase === "Stopped") return "Stopped";
+ return phases.length > 0 && phases.every((phase) => phase === "Stopped")
+ ? "Stopped"
+ : "Stopping";
+ }
+ if (phases.length === 0) return job.phase;
+ if (phases.some((phase) => phase === "Failed")) return "Failed";
+ if (phases.every((phase) => phase === "Succeeded")) return "Succeeded";
+ if (phases.every((phase) => phase === "Running")) return "Running";
+ if (job.phase === "Stopped" && phases.every((phase) => phase === "Stopped")) {
+ return "Stopped";
+ }
+ return "Pending";
+}
diff --git a/apps/rlark-ui/src/utils/nodeBatchMetadata.ts b/apps/rlark-ui/src/utils/nodeBatchMetadata.ts
index 9ace193..b8292c4 100644
--- a/apps/rlark-ui/src/utils/nodeBatchMetadata.ts
+++ b/apps/rlark-ui/src/utils/nodeBatchMetadata.ts
@@ -1,4 +1,11 @@
-import type { CRDNode } from "../types";
+import type { CRDNode, NodeCategory } from "../types";
+
+export const NODE_CATEGORY_KEYS = [
+ "rlark.io/node-category",
+ "rlark.io/node-category-cloud",
+ "rlark.io/node-category-edge",
+ "rlark.io/node-category-robot",
+] as const;
export type NodeModelMetadataUpdates = {
gpuModel?: string;
@@ -30,3 +37,28 @@ export function updateNodeModelMetadata(
applyUpdate("rlark.io/device-model", updates.deviceModel);
return { labels, annotations, removedLabelKeys, removedAnnotationKeys };
}
+
+export function updateNodeCategoryLabels(
+ labels: Record,
+ categories: NodeCategory[],
+) {
+ const next = { ...labels };
+ const removedKeys = new Set();
+ NODE_CATEGORY_KEYS.forEach((key) => {
+ delete next[key];
+ removedKeys.add(key);
+ });
+ (["cloud", "edge", "robot"] as const).forEach((category) => {
+ if (categories.includes(category)) {
+ next[`rlark.io/node-category-${category}`] = "true";
+ }
+ });
+ return { labels: next, removedKeys };
+}
+
+export function getRemovedLabelKeys(
+ original: Record,
+ next: Record,
+) {
+ return Object.keys(original).filter((key) => !(key in next));
+}
diff --git a/apps/rlark-ui/src/utils/nodeResources.ts b/apps/rlark-ui/src/utils/nodeResources.ts
index 210b3db..be676d5 100644
--- a/apps/rlark-ui/src/utils/nodeResources.ts
+++ b/apps/rlark-ui/src/utils/nodeResources.ts
@@ -14,6 +14,26 @@ function resourceNumber(value?: string): number {
return Number.isFinite(parsed) ? parsed : 0;
}
+export function selectDeviceResourceKey(
+ capacity: Record,
+ allocatable: Record,
+): string | undefined {
+ return Array.from(
+ new Set([...Object.keys(capacity), ...Object.keys(allocatable)]),
+ )
+ .filter(
+ (key) =>
+ (key === "rlinf.io/device" || key.startsWith("rlinf.io/device-")) &&
+ resourceNumber(capacity[key] ?? allocatable[key]) > 0,
+ )
+ .sort((left, right) => {
+ const leftGeneric = left === "rlinf.io/device";
+ const rightGeneric = right === "rlinf.io/device";
+ if (leftGeneric !== rightGeneric) return leftGeneric ? 1 : -1;
+ return left.localeCompare(right);
+ })[0];
+}
+
function formatResourceNumber(value: number): string {
return Number.isInteger(value) ? String(value) : value.toFixed(1);
}
diff --git a/apps/rlark-ui/src/utils/nodes.ts b/apps/rlark-ui/src/utils/nodes.ts
index 9f54e78..1ee8fee 100644
--- a/apps/rlark-ui/src/utils/nodes.ts
+++ b/apps/rlark-ui/src/utils/nodes.ts
@@ -21,6 +21,7 @@ export {
getGPUResourceKey,
getNodeResourceSummary,
parseResourceQuantity,
+ selectDeviceResourceKey,
} from "./nodeResources";
export function getNodeLocation(node: CRDNode): string {
diff --git a/apps/rlark-ui/src/utils/resourceAvailability.ts b/apps/rlark-ui/src/utils/resourceAvailability.ts
new file mode 100644
index 0000000..1715203
--- /dev/null
+++ b/apps/rlark-ui/src/utils/resourceAvailability.ts
@@ -0,0 +1,48 @@
+import type { CRDTask } from "../types";
+
+export type ReclaimableResources = Record>;
+
+function quantity(value?: string): number {
+ const parsed = Number.parseFloat(value ?? "0");
+ return Number.isFinite(parsed) ? parsed : 0;
+}
+
+export function availableResource(
+ allocatable: string | undefined,
+ used: string | undefined,
+ reclaimable = 0,
+): number {
+ const capacity = quantity(allocatable);
+ return Math.min(
+ capacity,
+ Math.max(0, capacity - quantity(used) + reclaimable),
+ );
+}
+
+export function reclaimableResourcesForTasks(
+ tasks: CRDTask[],
+): ReclaimableResources {
+ const reclaimable: ReclaimableResources = {};
+ for (const task of tasks) {
+ const requests =
+ task.spec?.kubernetes?.workload?.template.spec.containers?.reduce<
+ Record
+ >((totals, container) => {
+ for (const [key, value] of Object.entries(
+ container.resources?.requests ?? {},
+ )) {
+ totals[key] = (totals[key] ?? 0) + quantity(value);
+ }
+ return totals;
+ }, {}) ?? {};
+
+ for (const nodeName of task.status?.observedNodes ?? []) {
+ const nodeKey = `${task.metadata.namespace ?? ""}/${nodeName}`;
+ const nodeResources = (reclaimable[nodeKey] ??= {});
+ for (const [key, value] of Object.entries(requests)) {
+ nodeResources[key] = (nodeResources[key] ?? 0) + value;
+ }
+ }
+ }
+ return reclaimable;
+}
diff --git a/apps/rlark-ui/tests/job-actions.test.mjs b/apps/rlark-ui/tests/job-actions.test.mjs
new file mode 100644
index 0000000..344e93e
--- /dev/null
+++ b/apps/rlark-ui/tests/job-actions.test.mjs
@@ -0,0 +1,154 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+const jobsSource = await readFile(
+ new URL("../src/pages/Jobs.tsx", import.meta.url),
+ "utf8",
+);
+const jobsStyles = await readFile(
+ new URL("../src/styles.css", import.meta.url),
+ "utf8",
+);
+const createJobSource = await readFile(
+ new URL("../src/pages/CreateJob.tsx", import.meta.url),
+ "utf8",
+);
+const clustersSource = await readFile(
+ new URL("../src/pages/Clusters.tsx", import.meta.url),
+ "utf8",
+);
+const appSource = await readFile(
+ new URL("../src/App.tsx", import.meta.url),
+ "utf8",
+);
+
+test("failed jobs clean residual workers before starting", () => {
+ assert.match(
+ jobsSource,
+ /job\.phase === "Failed"\s*\? "clean-start"\s*:\s*"start"/,
+ );
+ assert.match(
+ jobsSource,
+ /action === "clean-start"\s*\? await handleCleanStart\(job\)/,
+ );
+ assert.match(jobsSource, /job\.phase === "Failed"/);
+ assert.match(
+ jobsSource,
+ /body: JSON\.stringify\(\{ spec: \{ stopped: true \} \}\)/,
+ );
+ assert.match(jobsSource, /await waitForFailedJobCleanup\(job\)/);
+ assert.match(
+ jobsSource,
+ /current\.phase === "Stopped" && current\.runningWorkers === 0/,
+ );
+ assert.match(
+ jobsSource,
+ /body: JSON\.stringify\(\{ spec: \{ stopped: false \} \}\)/,
+ );
+ assert.match(jobsSource, /isStartable \? : {
+ assert.match(jobsSource, /className="job-row-action"[\s\S]*?"复制"/);
+ assert.match(jobsSource, /className="job-row-action"[\s\S]*?"重启"/);
+ assert.match(jobsSource, /job-quick-lifecycle[\s\S]*?"停止"/);
+ assert.match(
+ jobsSource,
+ /data-tooltip=\{zh \? "更多操作" : "More actions"\}/,
+ );
+ assert.match(
+ jobsSource,
+ /className="action-dropdown-item danger"[\s\S]*?"删除"/,
+ );
+ assert.match(jobsSource, /onRestart=\{\(\) => setRestartTarget\(job\)\}/);
+ assert.match(jobsStyles, /\.job-row-action/);
+});
+
+test("job lifecycle actions use the shared in-app confirmation dialog", () => {
+ assert.doesNotMatch(jobsSource, /\bconfirm\(/);
+ assert.match(jobsSource, /function JobLifecycleConfirmDialog/);
+ assert.match(jobsSource, /className="modal-backdrop job-lifecycle-backdrop"/);
+ assert.match(jobsSource, /清理后启动任务?/);
+});
+
+test("worker event tooltip stays compact and shows only recent events", () => {
+ assert.match(jobsSource, /\.sort\(\(left, right\) =>/);
+ assert.match(jobsSource, /\.slice\(0, 4\)/);
+ assert.match(jobsSource, /recentEvents\.map/);
+ assert.match(jobsStyles, /-webkit-line-clamp: 2/);
+});
+
+test("long job IDs remain fully readable", () => {
+ assert.match(jobsSource, /job-id-cell/);
+ assert.match(jobsSource, /title=\{job\.id\}/);
+ assert.match(jobsStyles, /overflow-wrap: anywhere/);
+ assert.match(jobsStyles, /\.job-id-cell\.is-long strong/);
+});
+
+test("job deletion waits for worker cleanup before deleting", () => {
+ assert.match(jobsSource, /await waitForJobWorkersStopped\(job\)/);
+ assert.match(jobsSource, /task\.status\?\.phase === "Stopped"/);
+ assert.match(
+ jobsSource,
+ /await waitForJobWorkersStopped\(job\);[\s\S]*?method: "DELETE"/,
+ );
+});
+
+test("job actions report success and return to the list", () => {
+ assert.match(jobsSource, /className="job-action-notice" role="status"/);
+ assert.match(
+ jobsSource,
+ /setActionNotice\(zh \? "任务已删除" : "Job deleted"\)/,
+ );
+ assert.match(jobsSource, /if \(selectedName\) onSelect\(undefined\)/);
+ assert.match(jobsStyles, /\.job-action-notice/);
+});
+
+test("job submission reports success and returns to the job list", () => {
+ assert.match(createJobSource, /onSuccess: \(message: string\) => void/);
+ assert.match(createJobSource, /\? "任务提交成功"/);
+ assert.match(appSource, /setJobSubmitNotice\(message\)/);
+ assert.match(appSource, /navigate\("jobs", undefined, \{ replace: true \}\)/);
+ assert.match(
+ appSource,
+ /className="job-action-notice app-job-submit-notice"/,
+ );
+ assert.match(appSource, /role="status"/);
+});
+
+test("worker SSH copy is disabled when the jump host is unavailable", () => {
+ assert.match(jobsSource, /if \(!value\) return false/);
+ assert.match(jobsSource, /disabled=\{!sshCommand\}/);
+ assert.match(jobsSource, /未配置 SSH 跳板地址/);
+ assert.match(jobsSource, /\{sshCommand && \(/);
+});
+
+test("worker details show cluster and link node names", () => {
+ assert.match(
+ jobsSource,
+ /cluster: pod\.taskNamespace \|\| pod\.namespace \|\| "—"/,
+ );
+ assert.match(jobsSource, /zh \? "集群" : "Cluster"/);
+ assert.doesNotMatch(jobsSource, /zh \? "申请 CPU" : "CPU request"/);
+ assert.doesNotMatch(jobsSource, /zh \? "申请内存" : "Memory request"/);
+ assert.match(jobsSource, /function WorkerNodeLink/);
+ assert.match(jobsSource, /onClick=\{\(\) => onSelectNode\(node\)\}/);
+ assert.match(jobsStyles, /\.worker-node-link/);
+ assert.match(
+ clustersSource,
+ /realNodes\.find\(\(n\) => n\.metadata\.name === selectedNodeName\)/,
+ );
+});
+
+test("cloning a job without a node selector preserves automatic placement", () => {
+ assert.match(createJobSource, /sourceJob\.resources\s*\.filter/);
+ assert.match(
+ createJobSource,
+ /parseNodeSelectorStr\(resource\.nodeSelector\)/,
+ );
+ assert.match(
+ createJobSource,
+ /\.map\(\(resource\) => \[resource\.role, "model" as const\]\)/,
+ );
+});
diff --git a/apps/rlark-ui/tests/job-phase.test.mjs b/apps/rlark-ui/tests/job-phase.test.mjs
new file mode 100644
index 0000000..31cc9f1
--- /dev/null
+++ b/apps/rlark-ui/tests/job-phase.test.mjs
@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { effectiveJobPhase } from "../dist/test/utils/jobPhase.js";
+
+function job(phase, stopped, taskPhases) {
+ return {
+ phase,
+ stopped,
+ taskStatuses: taskPhases.map((taskPhase) => ({ phase: taskPhase })),
+ };
+}
+
+test("derives Running and Stopped only when every task matches", () => {
+ assert.equal(
+ effectiveJobPhase(job("Pending", false, ["Running", "Running"])),
+ "Running",
+ );
+ assert.equal(
+ effectiveJobPhase(job("Pending", true, ["Stopped", "Stopped"])),
+ "Stopped",
+ );
+ assert.equal(
+ effectiveJobPhase(job("Running", false, ["Running", "Pending"])),
+ "Pending",
+ );
+ assert.equal(
+ effectiveJobPhase(job("Running", true, ["Stopped", "Running"])),
+ "Stopping",
+ );
+});
+
+test("distinguishes stopping from normal Pending states", () => {
+ assert.equal(effectiveJobPhase(job("Pending", true, [])), "Stopping");
+ assert.equal(
+ effectiveJobPhase(job("Pending", true, ["Running", "Pending"])),
+ "Stopping",
+ );
+ assert.equal(
+ effectiveJobPhase(job("Pending", false, ["Running", "Pending"])),
+ "Pending",
+ );
+ assert.equal(effectiveJobPhase(job("Pending", false, [])), "Pending");
+});
+
+test("derives terminal states while the aggregate is Pending", () => {
+ assert.equal(
+ effectiveJobPhase(job("Pending", false, ["Pending", "Failed"])),
+ "Failed",
+ );
+ assert.equal(
+ effectiveJobPhase(job("Pending", false, ["Succeeded", "Succeeded"])),
+ "Succeeded",
+ );
+});
diff --git a/apps/rlark-ui/tests/login-page.test.mjs b/apps/rlark-ui/tests/login-page.test.mjs
new file mode 100644
index 0000000..1c15c7b
--- /dev/null
+++ b/apps/rlark-ui/tests/login-page.test.mjs
@@ -0,0 +1,43 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+const loginSource = await readFile(
+ new URL("../src/pages/Login.tsx", import.meta.url),
+ "utf8",
+);
+const adminLoginSource = await readFile(
+ new URL("../src/admin/AdminApp.tsx", import.meta.url),
+ "utf8",
+);
+
+test("user login shows the RLark brand logo", () => {
+ assert.match(loginSource, /src="\/rlark-logo-zh-light\.png"/);
+ assert.match(loginSource, /alt="RLark 具身智能云原生纳管平台"/);
+});
+
+test("password visibility control is accessible and does not submit", () => {
+ assert.match(loginSource, /type=\{showPassword \? "text" : "password"\}/);
+ assert.match(loginSource, /placeholder="请输入密码"/);
+ assert.doesNotMatch(loginSource, /placeholder="•+/);
+ assert.match(loginSource, /type="button"/);
+ assert.match(loginSource, /aria-label=\{showPassword \? "隐藏密码" : "显示密码"\}/);
+ assert.match(loginSource, /aria-pressed=\{showPassword\}/);
+});
+
+test("login errors render inline below the credentials", () => {
+ assert.match(loginSource, /className="admin-login-panel"/);
+ assert.match(loginSource, /className="login-inline-error"/);
+ assert.match(loginSource, /role="alert"/);
+ assert.match(loginSource, /window\.setTimeout\(\(\) => setError\(""\), 4000\)/);
+});
+
+test("admin login shares password and inline error controls", () => {
+ assert.doesNotMatch(adminLoginSource, /className="admin-login-topbar"/);
+ assert.match(adminLoginSource, /className="admin-login-panel"/);
+ assert.match(adminLoginSource, /className="admin-login-badge">ADMIN/);
+ assert.match(adminLoginSource, /type=\{showPassword \? "text" : "password"\}/);
+ assert.match(adminLoginSource, /placeholder=\{zh \? "请输入密码" : "Enter password"\}/);
+ assert.match(adminLoginSource, /className="login-inline-error"/);
+ assert.match(adminLoginSource, /window\.setTimeout\(\(\) => setError\(""\), 4000\)/);
+});
diff --git a/apps/rlark-ui/tests/node-batch-metadata.test.mjs b/apps/rlark-ui/tests/node-batch-metadata.test.mjs
index 60b9206..c478191 100644
--- a/apps/rlark-ui/tests/node-batch-metadata.test.mjs
+++ b/apps/rlark-ui/tests/node-batch-metadata.test.mjs
@@ -1,6 +1,11 @@
import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
import test from "node:test";
-import { updateNodeModelMetadata } from "../dist/test/utils/nodeBatchMetadata.js";
+import {
+ getRemovedLabelKeys,
+ updateNodeCategoryLabels,
+ updateNodeModelMetadata,
+} from "../dist/test/utils/nodeBatchMetadata.js";
function node(category, labels = {}, annotations = {}) {
return {
@@ -41,3 +46,38 @@ test("blank values remove the selected model annotation", () => {
assert.equal(result.labels["rlark.io/gpu-model"], undefined);
assert.equal(result.removedAnnotationKeys.has("rlark.io/gpu-model"), true);
});
+
+test("removes deselected categories from a multi-category node", () => {
+ const result = updateNodeCategoryLabels(
+ {
+ "rlark.io/node-category-edge": "true",
+ "rlark.io/node-category-robot": "true",
+ },
+ ["edge"],
+ );
+ assert.equal(result.labels["rlark.io/node-category-edge"], "true");
+ assert.equal(result.labels["rlark.io/node-category-robot"], undefined);
+ assert.equal(result.removedKeys.has("rlark.io/node-category-robot"), true);
+});
+
+test("detects labels removed in the detail editor", () => {
+ assert.deepEqual(
+ getRemovedLabelKeys({ zone: "a", team: "robot" }, { zone: "a" }),
+ ["team"],
+ );
+});
+
+test("node management keeps selection separate from detail navigation", () => {
+ const browserSource = readFileSync(
+ new URL("../src/components/NodeResourceBrowser.tsx", import.meta.url),
+ "utf8",
+ );
+ const adminSource = readFileSync(
+ new URL("../src/admin/AdminPage.tsx", import.meta.url),
+ "utf8",
+ );
+
+ assert.match(browserSource, /className="node-row-primary node-detail-link"/);
+ assert.doesNotMatch(browserSource, /role="button"\s+tabIndex=\{0\}/);
+ assert.match(adminSource, /className="admin-node-overview"/);
+});
diff --git a/apps/rlark-ui/tests/node-resources.test.mjs b/apps/rlark-ui/tests/node-resources.test.mjs
index 51434b5..d84298c 100644
--- a/apps/rlark-ui/tests/node-resources.test.mjs
+++ b/apps/rlark-ui/tests/node-resources.test.mjs
@@ -1,6 +1,31 @@
import assert from "node:assert/strict";
import test from "node:test";
-import { getNodeResourceSummary } from "../dist/test/utils/nodeResources.js";
+import {
+ getNodeResourceSummary,
+ selectDeviceResourceKey,
+} from "../dist/test/utils/nodeResources.js";
+
+test("prefers a positive modeled device resource over zero-capacity keys", () => {
+ const capacity = {
+ "rlinf.io/device": "0",
+ "rlinf.io/device-franka": "1",
+ "rlinf.io/device-macvlan": "0",
+ };
+ assert.equal(
+ selectDeviceResourceKey(capacity, capacity),
+ "rlinf.io/device-franka",
+ );
+});
+
+test("uses the positive generic device resource only as a fallback", () => {
+ assert.equal(
+ selectDeviceResourceKey(
+ { "rlinf.io/device": "1", "rlinf.io/device-franka": "0" },
+ { "rlinf.io/device": "1", "rlinf.io/device-franka": "0" },
+ ),
+ "rlinf.io/device",
+ );
+});
test("shows GPU and embodied device resources together on an edge node", () => {
const summary = getNodeResourceSummary(
diff --git a/apps/rlark-ui/tests/resource-availability.test.mjs b/apps/rlark-ui/tests/resource-availability.test.mjs
new file mode 100644
index 0000000..80c45ee
--- /dev/null
+++ b/apps/rlark-ui/tests/resource-availability.test.mjs
@@ -0,0 +1,85 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ availableResource,
+ reclaimableResourcesForTasks,
+} from "../dist/test/utils/resourceAvailability.js";
+
+const task = (namespace, nodes, gpu) => ({
+ metadata: { name: "job-actor", namespace },
+ spec: {
+ kubernetes: {
+ workload: {
+ kind: "StatefulSet",
+ replicas: nodes.length,
+ template: {
+ spec: {
+ containers: [
+ {
+ name: "main",
+ image: "example/image",
+ env: [],
+ resources: { requests: { "nvidia.com/gpu": gpu } },
+ },
+ ],
+ },
+ },
+ },
+ },
+ },
+ status: { observedNodes: nodes },
+});
+
+test("keeps normal availability unchanged without reclaimable resources", () => {
+ assert.equal(availableResource("8", "8"), 0);
+});
+
+test("restores the current task GPU allocation when restarting", () => {
+ const reclaimable = reclaimableResourcesForTasks([
+ task("cluster-a", ["node-a"], "2"),
+ ]);
+ assert.equal(reclaimable["cluster-a/node-a"]["nvidia.com/gpu"], 2);
+ assert.equal(
+ availableResource(
+ "8",
+ "2",
+ reclaimable["cluster-a/node-a"]["nvidia.com/gpu"],
+ ),
+ 8,
+ );
+});
+
+test("does not restore resources occupied by other tasks", () => {
+ const reclaimable = reclaimableResourcesForTasks([
+ task("cluster-a", ["node-a", "node-b"], "2"),
+ ]);
+ assert.equal(
+ availableResource(
+ "8",
+ "6",
+ reclaimable["cluster-a/node-a"]["nvidia.com/gpu"],
+ ),
+ 4,
+ );
+ assert.equal(
+ availableResource(
+ "8",
+ "6",
+ reclaimable["cluster-a/node-b"]["nvidia.com/gpu"],
+ ),
+ 4,
+ );
+});
+
+test("caps restored availability at allocatable capacity", () => {
+ assert.equal(availableResource("8", "1", 4), 8);
+});
+
+test("keeps nodes with the same name in different clusters separate", () => {
+ const reclaimable = reclaimableResourcesForTasks([
+ task("cluster-a", ["worker"], "1"),
+ task("cluster-b", ["worker"], "2"),
+ ]);
+ assert.equal(reclaimable["cluster-a/worker"]["nvidia.com/gpu"], 1);
+ assert.equal(reclaimable["cluster-b/worker"]["nvidia.com/gpu"], 2);
+});
diff --git a/apps/rlark-ui/tests/resource-placement.test.mjs b/apps/rlark-ui/tests/resource-placement.test.mjs
new file mode 100644
index 0000000..13e5f73
--- /dev/null
+++ b/apps/rlark-ui/tests/resource-placement.test.mjs
@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+const pickerSource = await readFile(
+ new URL("../src/components/ResourcePlacementPicker.tsx", import.meta.url),
+ "utf8",
+);
+
+test("lists nodes without discovered device resources", () => {
+ assert.match(pickerSource, /else if \(resources\.length === 0\)/);
+ assert.match(pickerSource, /model: "__undiscovered_device__"/);
+ assert.match(pickerSource, /resourceKey: ""/);
+ assert.match(pickerSource, /"未分类"/);
+ assert.match(pickerSource, /"未发现设备"/);
+});
+
+test("shows a truthful placeholder when a device resource has no model", () => {
+ assert.match(pickerSource, /const deviceModel = getNodeDeviceModel\(fullNode\)/);
+ assert.match(pickerSource, /resourceKey: deviceModel \? deviceKey : ""/);
+ assert.match(pickerSource, /configured: Boolean\(deviceModel\)/);
+ assert.match(pickerSource, /"未设置设备型号"/);
+ assert.match(
+ pickerSource,
+ /仅按未设置设备型号的节点调度,不申请设备资源/,
+ );
+ assert.match(pickerSource, /resourceAmount} × \$\{displayModel}/);
+});
+
+test("unconfigured devices constrain nodes without requesting fake resources", () => {
+ assert.match(pickerSource, /!resource\.configured \|\| resource\.free >= nextPerWorker/);
+ assert.match(pickerSource, /candidates\[0\]\?\.resourceKey/);
+ assert.match(pickerSource, /disabled=\{isUnconfiguredDevice\}/);
+ assert.match(
+ pickerSource,
+ /仅按未分类节点调度,不申请未发现的设备资源/,
+ );
+});
+
+test("edit mode restores the original resource and selected nodes", () => {
+ assert.match(pickerSource, /nodeSelector: string/);
+ assert.match(pickerSource, /parseNodeSelectorStr\(nodeSelector\)/);
+ assert.match(pickerSource, /restoresAutomaticMode/);
+ assert.match(pickerSource, /placementMode \?\?/);
+ assert.match(pickerSource, /onPlacementModeChange\?\.\("manual"\)/);
+ assert.match(pickerSource, /selectedNames\.size === matchingNames\.length/);
+ assert.match(pickerSource, /setMode\("manual"\)/);
+ assert.match(pickerSource, /selectedNames\.has\(name\)/);
+ assert.match(pickerSource, /initializedRef\.current = true/);
+});
+
+test("new placement defaults to manual node selection", () => {
+ assert.match(
+ pickerSource,
+ /useState\(placementMode \?\? "manual"\)/,
+ );
+});
+
+test("node cards keep long node names readable", () => {
+ assert.match(pickerSource, /title=\{name\}/);
+ assert.match(pickerSource, /className="placement-node-meta"/);
+ assert.match(pickerSource, /className="placement-node-state"/);
+});
diff --git a/apps/rlark/docs/user-guide/jobs.md b/apps/rlark/docs/user-guide/jobs.md
index d566b9b..0b6f3bc 100644
--- a/apps/rlark/docs/user-guide/jobs.md
+++ b/apps/rlark/docs/user-guide/jobs.md
@@ -12,7 +12,11 @@ Platform Console → Jobs → Create Job. Enter a name and define the Worker rol
2. Choose one GPU or embodied-device specification from the list, which shows available/total devices and nodes, then set the shared per-Worker resource request. Set the request to `0` for debugging workloads that should keep the placement constraint without requesting the selected device.
3. Choose one scheduling mode:
- **Automatic selection**: enter the desired Worker count. The console validates the total request against schedulable capacity and selects eligible nodes.
- - **Select nodes**: click eligible nodes or drag across node cards. Each selected node creates one Worker; click or drag across selected nodes again to remove them.
+- **Select nodes**: click eligible nodes or drag across node cards. Each selected node creates one Worker; click or drag across selected nodes again to remove them.
+
+When cloning a Job, the scheduling mode is preserved. A role that did not pin
+specific nodes remains in automatic-selection mode instead of being converted
+to manual node selection.
4. Review the shared placement summary, then configure the image, prepare script, environment variables, and storage mounts.
Submit the Job, then open Job details to verify the running state and inspect Workers.
@@ -50,9 +54,9 @@ For each worker role, configure the following:
- **Init Scripts** — Commands that run before the main entrypoint. Useful for environment setup, dependency installation, or data preparation.
-- **Storage Mounts** — Attach persistent storage to worker containers. Two types are supported:
- - **hostPath**: Mount a directory from the host node's filesystem. Data persists on the node after the job is deleted.
- - **PVC** (PersistentVolumeClaim): Mount a Kubernetes persistent volume. PVCs are automatically cleaned up when the job is deleted.
+- **Storage Mounts** — Attach storage to worker containers. Two types are supported:
+ - **hostPath**: Mount a directory from the host node's filesystem. Its data is not deleted by Job lifecycle actions.
+ - **PVC** (PersistentVolumeClaim): Mount a Kubernetes persistent volume. Stopping, restarting, or deleting the Job deletes its task PVCs; starting or restarting creates empty PVCs.

@@ -90,6 +94,8 @@ Below the overview, the worker list shows every worker instance with:
- **Instance Name** — Auto-generated name (e.g., `myjob-actor-0`)
- **Role** — The role this worker belongs to
- **Node** — The physical node hosting this worker
+- **Cluster** — The data-plane cluster that owns the Worker; the node name is a
+ link to that node's detail page
- **IP** — The Pod IP address
- **Status** — Per-worker status (Pending, ContainerCreating, Running, Terminated)
@@ -150,6 +156,8 @@ nvidia-smi # Check GPU status (if GPU is allocated)
### File Transfer
+The key button in the Worker list copies the SSH connection command. It requires an administrator-configured SSH jump host; when that setting is missing, the button is disabled and explains why instead of copying empty content.
+
The WebTerminal supports file upload and download:
- **Upload** — Upload files from your local machine to the container's default working directory. File names must match the pattern `[A-Za-z0-9._-]+`.
@@ -165,25 +173,30 @@ The detail-page action bar supports the full Job lifecycle:
- **Stop** pauses a running Job and preserves its configuration.
- **Start** resumes a stopped Job.
-- **Restart** opens a choice: restart immediately with the current configuration, or edit the Job and restart after the updated configuration is saved.
+- **Restart** opens a choice: restart immediately with the current configuration, or edit the Job and restart after the updated configuration is saved. During edit-and-restart, available capacity includes resources that the current Job will release.
- **Delete** opens a danger confirmation that identifies the target Job and warns that the operation cannot be undone before permanently removing it.
-Lifecycle actions require confirmation. While an action is in progress, the other action buttons are disabled; failures are shown in the same action area. A Job is removed from the page only after the delete request succeeds.
+Lifecycle actions require confirmation. While an action is in progress, the
+other action buttons are disabled; failures are shown in the same action area.
+After a successful action, the console returns to the Jobs list and shows a
+completion notice.
-The Jobs table provides a Start/Stop shortcut in each row. Open the adjacent actions menu to clone, restart, or delete the Job; Restart uses the same immediate-restart or edit-and-restart choice as the detail page.
+Deletion is synchronous from the console's perspective: RLark first stops the
+Job, waits until all child Tasks and Workers reach a terminal state, deletes the
+Job, and then waits until the Job and child Tasks are gone before updating the
+list.
-### Stop a Running Job
+Each Jobs table row directly exposes Clone, Restart, and Start/Stop text buttons, while Delete remains in the adjacent more-actions menu. Restart uses the same immediate-restart or edit-and-restart choice as the detail page. Expanded Worker details focus on cluster, role, node, and GPU information without CPU or memory request cards.
-Stopping a job gracefully terminates all worker Pods while preserving:
+### Stop a Running Job
-- PVC data (persistent volumes remain intact)
-- Job logs and metadata
+Stopping a Job terminates all Worker Pods and deletes its task PVCs. Job configuration, logs, metadata, and hostPath data are preserved.
-You can resume a stopped job later without losing state.
+The time at which a manually stopped Job enters `Stopped` is recorded and shown in the Jobs table.
### Resume a Stopped Job
-Resuming a job recreates the worker Pods from the same configuration. PVCs are reattached, so data stored on persistent volumes is available immediately.
+Starting a stopped Job recreates the Worker Pods and empty task PVCs from the same configuration. Previous PVC data is not restored; hostPath data remains available.
### Delete a Job
@@ -222,4 +235,4 @@ All job management operations can be performed programmatically via the REST API
POST /api/v1/rlinf.io/v1alpha1/jobs
```
-This endpoint accepts a `Job` CRD object in the request body. For complete request examples and status query patterns, see [API Examples](../api/examples.md).
\ No newline at end of file
+This endpoint accepts a `Job` CRD object in the request body. For complete request examples and status query patterns, see [API Examples](../api/examples.md).
diff --git a/apps/rlark/docs/user-guide/ssh-keys.md b/apps/rlark/docs/user-guide/ssh-keys.md
index 124c70e..f6bacbd 100644
--- a/apps/rlark/docs/user-guide/ssh-keys.md
+++ b/apps/rlark/docs/user-guide/ssh-keys.md
@@ -28,7 +28,7 @@ This selection writes one public key into the Job configuration for workload inj
After an administrator configures the Server SSH endpoint, use the command displayed by the console or copy the Worker-specific command from Job or Node details. A typical Worker command is:
```bash
-ssh -J @: root@
+ssh -J : root@
```
The outer SSH username must match the username attached to the registered key. The target container must provide its SSH service and accept the selected key.
diff --git a/apps/rlark/docs/user-guide/storage.md b/apps/rlark/docs/user-guide/storage.md
index 5bddb3a..f3d5499 100644
--- a/apps/rlark/docs/user-guide/storage.md
+++ b/apps/rlark/docs/user-guide/storage.md
@@ -4,10 +4,10 @@
RLark supports two storage types for training jobs:
-| Type | Use Case | Persistence |
-|------|----------|-------------|
-| Host Directory | Data already on the node, high I/O | Survives pod restarts |
-| Object Storage (PVC) | Shared data, checkpoint persistence | Survives pod deletion |
+| Type | Use Case | Lifecycle |
+|------|----------|-----------|
+| Host Directory | Data already on the node, high I/O | Job lifecycle actions do not delete data |
+| Object Storage (PVC) | Shared data within a Job run | Stop/restart/delete removes task PVCs; start/restart creates empty PVCs |
## Host Directory
@@ -39,12 +39,13 @@ Verify the storage chain:
2. Container mount is correct
3. Application can read input data
4. Application can write output data
-5. Output persists after job stops
+5. Copy required PVC output elsewhere before stopping or restarting the Job
-## Cleanup
+## Lifecycle
-- Stopping a job: PVCs are preserved, hostPath data is preserved
-- Deleting a job: PVCs are cleaned up, hostPath data is NOT cleaned up
+- Stop or restart: task PVCs are deleted; hostPath data is preserved
+- Start or restart: new empty task PVCs are created
+- Delete: task PVCs are deleted; hostPath data is preserved
## API Equivalent
diff --git a/apps/rlark/docs/zh/user-guide/jobs.md b/apps/rlark/docs/zh/user-guide/jobs.md
index dcbd4cf..9f51fc7 100644
--- a/apps/rlark/docs/zh/user-guide/jobs.md
+++ b/apps/rlark/docs/zh/user-guide/jobs.md
@@ -13,6 +13,8 @@ Job 是面向用户的工作负载。创建任务时选择模板或配置 Task
3. 选择一种调度方式:
- **自动选择**:填写需要创建的 Worker 数量,控制台会按可调度容量校验总申请量并自动选择符合条件的节点。
- **指定节点**:单击符合条件的节点或拖拽框选,每个选中节点创建一个 Worker;再次单击或框选已选节点可取消选择。
+
+复制任务时会保留原任务的调度方式。未指定具体节点的角色仍使用自动选择,不会被转换为指定节点。
4. 检查统一的资源总结,然后配置镜像、准备脚本、环境变量和存储挂载。
提交任务后,在任务详情页确认状态进入运行中并查看 Worker。
@@ -50,9 +52,9 @@ RLark 支持以下任务类型,每种类型预配置了适合该工作负载
- **准备脚本** — 在主入口命令之前运行的命令。用于环境设置、依赖安装或数据准备。
-- **存储挂载** — 向 Worker 容器挂载持久存储。支持两种类型:
- - **hostPath**:挂载宿主机节点上的目录。任务删除后数据保留在节点上。
- - **PVC**(PersistentVolumeClaim):挂载 Kubernetes 持久卷。任务删除时 PVC 自动清理。
+- **存储挂载** — 向 Worker 容器挂载存储。支持两种类型:
+ - **hostPath**:挂载宿主机节点上的目录,任务生命周期操作不会删除其中的数据。
+ - **PVC**(PersistentVolumeClaim):挂载 Kubernetes 持久卷。停止、重启或删除任务会删除任务 PVC;启动或重启会新建空 PVC。

@@ -90,6 +92,7 @@ RLark 支持以下任务类型,每种类型预配置了适合该工作负载
- **实例名称** — 自动生成的名称(如 `myjob-actor-0`)
- **角色** — 该 Worker 所属角色
- **节点** — 承载该 Worker 的物理节点
+- **集群** — Worker 所属的数据面集群;节点名称可点击并跳转到对应节点详情页
- **IP** — Pod IP 地址
- **状态** — 每个 Worker 的状态(Pending、ContainerCreating、Running、Terminated)
@@ -150,6 +153,8 @@ nvidia-smi # 检查 GPU 状态(如有 GPU 分配)
### 文件传输
+Worker 列表中的钥匙按钮用于复制 SSH 连接命令。该功能依赖管理员配置 SSH 跳板地址;未配置时按钮会禁用并提示原因,避免复制空内容。
+
WebTerminal 支持文件上传和下载:
- **上传** — 从本地机器上传文件到容器默认工作目录。文件名需匹配 `[A-Za-z0-9._-]+` 模式。
@@ -165,25 +170,24 @@ WebTerminal 支持文件上传和下载:
- **停止任务**:暂停运行中的任务并保留现有配置。
- **启动任务**:恢复已停止的任务。
-- **重启任务**:选择使用当前配置一键重启,或先编辑任务配置并在保存后自动重启。
+- **重启任务**:选择使用当前配置一键重启,或先编辑任务配置并在保存后自动重启。编辑后重启时,资源可用量会包含当前任务即将释放的资源。
- **删除任务**:在危险操作弹窗中核对任务名称与不可恢复提示后,永久删除任务。
-执行生命周期操作前需要确认。请求处理中会禁用其他操作按钮,失败信息会显示在操作区;只有删除请求成功后页面才会返回任务列表。
+执行生命周期操作前需要确认。请求处理中会禁用其他操作按钮,失败信息会显示在操作区;操作成功后页面会返回任务列表并显示完成提示。
-任务列表的每一行都提供启动/停止快捷按钮;相邻的更多操作菜单包含复制、重启和删除,其中重启与详情页一样支持"一键重启"或"编辑后重启"。
+控制台以同步流程执行删除:先停止任务,等待所有子 Task 和 Worker 进入终态,再删除 Job,并继续等待 Job 与子 Task 均被清理后才更新列表。
-### 停止运行中的任务
+任务列表的每一行直接提供复制、重启和启动/停止文字按钮,删除操作收纳在相邻的更多操作菜单中;重启与详情页一样支持“一键重启”或“编辑后重启”。Worker 展开详情聚焦集群、角色、节点和 GPU 信息,不展示 CPU 与内存申请卡片。
-停止任务会优雅地终止所有 Worker Pod,同时保留:
+### 停止运行中的任务
-- PVC 数据(持久卷保持不变)
-- 任务日志和元数据
+停止任务会终止所有 Worker Pod 并删除任务 PVC。任务配置、日志、元数据和 hostPath 数据会保留。
-停止后可恢复任务,不丢失状态。
+手动停止的任务进入 `Stopped` 状态时会记录停止时间,并显示在任务列表中。
### 恢复已停止的任务
-恢复任务会从相同配置重新创建 Worker Pod。PVC 重新挂载,持久卷上的数据立即可用。
+启动已停止的任务会按相同配置重新创建 Worker Pod 和空的任务 PVC。原 PVC 数据不会恢复,hostPath 数据仍然可用。
### 删除任务
@@ -222,4 +226,4 @@ WebTerminal 支持文件上传和下载:
POST /api/v1/rlinf.io/v1alpha1/jobs
```
-该端点接受请求体中的 `Job` CRD 对象。完整请求示例和状态查询模式参见 [API 示例](../api/examples.md)。
\ No newline at end of file
+该端点接受请求体中的 `Job` CRD 对象。完整请求示例和状态查询模式参见 [API 示例](../api/examples.md)。
diff --git a/apps/rlark/docs/zh/user-guide/ssh-keys.md b/apps/rlark/docs/zh/user-guide/ssh-keys.md
index 6baa23b..276a19e 100644
--- a/apps/rlark/docs/zh/user-guide/ssh-keys.md
+++ b/apps/rlark/docs/zh/user-guide/ssh-keys.md
@@ -28,7 +28,7 @@ RLark 会校验公钥格式并拒绝重复密钥。当 Server SSH 地址可用
管理员配置 Server SSH 入口后,使用控制台显示的命令,或从 Job/节点详情复制指定 Worker 的命令。典型命令如下:
```bash
-ssh -J @: root@
+ssh -J : root@
```
外层 SSH 用户名必须与公钥登记的用户名一致。目标容器必须提供 SSH 服务,并接受所选公钥。
diff --git a/apps/rlark/docs/zh/user-guide/storage.md b/apps/rlark/docs/zh/user-guide/storage.md
index b942b6c..09bdfc9 100644
--- a/apps/rlark/docs/zh/user-guide/storage.md
+++ b/apps/rlark/docs/zh/user-guide/storage.md
@@ -4,10 +4,10 @@
RLark 支持两种存储类型:
-| 类型 | 适用场景 | 持久性 |
-|------|----------|--------|
-| 主机目录 | 数据已在节点上,高 I/O | Pod 重启后保留 |
-| 对象存储(PVC) | 共享数据、checkpoint 持久化 | Pod 删除后保留 |
+| 类型 | 适用场景 | 生命周期 |
+|------|----------|----------|
+| 主机目录 | 数据已在节点上,高 I/O | 任务生命周期操作不删除数据 |
+| 对象存储(PVC) | 单次任务运行内共享数据 | 停止/重启/删除会删除任务 PVC;启动/重启会新建空 PVC |
## 主机目录
@@ -39,12 +39,13 @@ RLark 支持两种存储类型:
2. 容器挂载正确
3. 应用可读取输入数据
4. 应用可写入输出数据
-5. 任务停止后输出持久存在
+5. 停止或重启任务前,将需保留的 PVC 输出复制到其他位置
-## 清理
+## 生命周期
-- 停止任务:PVC 保留,hostPath 数据保留
-- 删除任务:PVC 清理,hostPath 数据不清理
+- 停止或重启:删除任务 PVC,保留 hostPath 数据
+- 启动或重启:新建空的任务 PVC
+- 删除:删除任务 PVC,保留 hostPath 数据
## API 等效操作
diff --git a/apps/rlark/pkg/addons/catalog/csi-driver-rclone/manifests/controller.yaml b/apps/rlark/pkg/addons/catalog/csi-driver-rclone/manifests/controller.yaml
index 45560fe..fe805c4 100644
--- a/apps/rlark/pkg/addons/catalog/csi-driver-rclone/manifests/controller.yaml
+++ b/apps/rlark/pkg/addons/catalog/csi-driver-rclone/manifests/controller.yaml
@@ -43,6 +43,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
+ securityContext:
+ privileged: true
volumeMounts:
- name: socket-dir
mountPath: /csi
@@ -78,4 +80,4 @@ spec:
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet
- type: Directory
\ No newline at end of file
+ type: Directory
diff --git a/apps/rlark/pkg/addons/catalog_test.go b/apps/rlark/pkg/addons/catalog_test.go
new file mode 100644
index 0000000..3d8c228
--- /dev/null
+++ b/apps/rlark/pkg/addons/catalog_test.go
@@ -0,0 +1,42 @@
+package addons
+
+import (
+ "testing"
+
+ "go.yaml.in/yaml/v3"
+)
+
+func TestRcloneControllerIsPrivilegedForBidirectionalMounts(t *testing.T) {
+ addon, ok := Registry.Get("csi-driver-rclone")
+ if !ok {
+ t.Fatal("csi-driver-rclone addon is not registered")
+ }
+
+ manifests, err := addon.Render(nil, "rlark-system", "csi-driver-rclone", "test-uid")
+ if err != nil {
+ t.Fatalf("render addon: %v", err)
+ }
+
+ for _, manifest := range manifests {
+ var resource map[string]any
+ if err := yaml.Unmarshal(manifest.Raw, &resource); err != nil {
+ t.Fatalf("unmarshal rendered manifest: %v", err)
+ }
+ if resource["kind"] != "Deployment" {
+ continue
+ }
+
+ spec := resource["spec"].(map[string]any)
+ template := spec["template"].(map[string]any)
+ podSpec := template["spec"].(map[string]any)
+ containers := podSpec["containers"].([]any)
+ rclone := containers[0].(map[string]any)
+ securityContext := rclone["securityContext"].(map[string]any)
+ if privileged, _ := securityContext["privileged"].(bool); !privileged {
+ t.Fatal("rclone controller must be privileged for Bidirectional mount propagation")
+ }
+ return
+ }
+
+ t.Fatal("rclone controller Deployment was not rendered")
+}
diff --git a/apps/rlark/pkg/agent/container/network.go b/apps/rlark/pkg/agent/container/network.go
index dd8ffe1..b05b1c1 100644
--- a/apps/rlark/pkg/agent/container/network.go
+++ b/apps/rlark/pkg/agent/container/network.go
@@ -18,6 +18,7 @@ import (
"github.com/rlinf/rlark/apps/rlark/pkg/apis"
"github.com/rlinf/rlark/apps/rlark/pkg/common"
"github.com/rlinf/rlark/apps/rlark/pkg/log"
+ nodeservermetrics "github.com/rlinf/rlark/apps/rlark/pkg/network/nodeserver"
"github.com/rlinf/rlark/apps/rlark/pkg/utils"
)
@@ -99,6 +100,7 @@ func NewContainerNetworkAdapter(
sshAddr: sshAddr,
sshDialer: NewSSHDialer(SSHDialerConfig{
HostKeyCallback: hostKeyCallback,
+ OnReconnect: nodeservermetrics.OnReconnect(),
}),
enableSameClusterDirect: enableSameClusterDirect,
enableCrossClusterDirect: enableCrossClusterDirect,
@@ -214,7 +216,13 @@ func (a *containerNetworkAdapter) GetContainerNetworkDial(ctx context.Context, c
return func(ctx context.Context) (net.Conn, error) {
var dialer net.Dialer
// 直接通过目标 Pod 的 LocalIP 访问其 5700 端口(proxy 端口)
- return dialer.DialContext(ctx, "tcp", net.JoinHostPort(targetPod.LocalIP, "5700"))
+ conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(targetPod.LocalIP, "5700"))
+ status := "success"
+ if err != nil {
+ status = "error"
+ }
+ nodeservermetrics.Metrics().IncDial("direct", status)
+ return conn, err
}, nil
}
@@ -228,7 +236,13 @@ func (a *containerNetworkAdapter) GetContainerNetworkDial(ctx context.Context, c
target := fmt.Sprintf("%s.%s.%s.agent-node:5700", targetPod.LocalIP, targetPod.Node, agentID)
logger.V(1).Info("Target pod is in a different cluster, using control plane proxy", "targetPod", target)
return func(ctx context.Context) (net.Conn, error) {
- return a.sshDialer.DialContext(ctx, cred.DomainID, a.sshAddr, dpeer.Spec.Cert, dpeer.Spec.Key, target)
+ conn, err := a.sshDialer.DialContext(ctx, cred.DomainID, a.sshAddr, dpeer.Spec.Cert, dpeer.Spec.Key, target)
+ status := "success"
+ if err != nil {
+ status = "error"
+ }
+ nodeservermetrics.Metrics().IncDial("ssh", status)
+ return conn, err
}, nil
}
diff --git a/apps/rlark/pkg/agent/container/ssh_dialer.go b/apps/rlark/pkg/agent/container/ssh_dialer.go
index 08cc6e7..6c371d6 100644
--- a/apps/rlark/pkg/agent/container/ssh_dialer.go
+++ b/apps/rlark/pkg/agent/container/ssh_dialer.go
@@ -10,9 +10,11 @@ import (
"strings"
"sync"
"sync/atomic"
+ "syscall"
"time"
"github.com/rlinf/rlark/apps/rlark/pkg/auth/cert"
+ "github.com/rlinf/rlark/apps/rlark/pkg/log"
"golang.org/x/crypto/ssh"
)
@@ -28,17 +30,20 @@ import (
// ---------------------------------------------------------------------------
const (
- defaultIdleTimeout = 10 * time.Minute
- defaultCleanupInterval = 1 * time.Minute
- defaultSSHUser = "root"
- defaultSSHTimeout = 10 * time.Second
- maxReconnectBackoff = 30 * time.Second
- initialReconnectBackoff = 1 * time.Second
+ defaultIdleTimeout = 24 * time.Hour
+ defaultCleanupInterval = 1 * time.Minute
+ defaultSSHUser = "root"
+ defaultSSHTimeout = 10 * time.Second
+ defaultKeepaliveInterval = 30 * time.Second
+ maxReconnectBackoff = 30 * time.Second
+ initialReconnectBackoff = 1 * time.Second
)
+const keepaliveRequest = "keepalive@openssh.com"
+
// SSHDialerConfig 配置全局 SSH 连接管理器。
type SSHDialerConfig struct {
- // IdleTimeout 关闭空闲超过此时长的 SSH 连接。零值使用默认值(10 分钟)。
+ // IdleTimeout 关闭空闲超过此时长的 SSH 连接。零值使用默认值(24 小时)。
IdleTimeout time.Duration `json:"idleTimeout,omitempty" yaml:"idleTimeout,omitempty"`
// CleanupInterval 垃圾回收周期。零值使用默认值(1 分钟)。
CleanupInterval time.Duration `json:"cleanupInterval,omitempty" yaml:"cleanupInterval,omitempty"`
@@ -50,6 +55,10 @@ type SSHDialerConfig struct {
InitialReconnectBackoff time.Duration `json:"initialReconnectBackoff,omitempty" yaml:"initialReconnectBackoff,omitempty"`
// MaxReconnectBackoff 重连等待时间的上限。零值使用默认值(30 秒)。
MaxReconnectBackoff time.Duration `json:"maxReconnectBackoff,omitempty" yaml:"maxReconnectBackoff,omitempty"`
+ // KeepaliveInterval 应用层 SSH 保活间隔。零值使用默认值(30 秒)。
+ KeepaliveInterval time.Duration `json:"keepaliveInterval,omitempty" yaml:"keepaliveInterval,omitempty"`
+ // OnReconnect 重连成功后的回调(用于 metrics 埋点)。可为 nil。
+ OnReconnect func(domainID string) `json:"-" yaml:"-"`
// HostKeyCallback SSH 主机密钥验证回调。nil 时使用 InsecureIgnoreHostKey(仅开发环境)。
HostKeyCallback ssh.HostKeyCallback `json:"-" yaml:"-"`
}
@@ -73,6 +82,9 @@ func (c *SSHDialerConfig) setDefaults() {
if c.MaxReconnectBackoff <= 0 {
c.MaxReconnectBackoff = maxReconnectBackoff
}
+ if c.KeepaliveInterval <= 0 {
+ c.KeepaliveInterval = defaultKeepaliveInterval
+ }
if c.HostKeyCallback == nil {
c.HostKeyCallback = ssh.InsecureIgnoreHostKey()
}
@@ -94,6 +106,10 @@ type domainEntry struct {
lastReconnectAt time.Time
reconnectBackoff time.Duration
maxBackoff time.Duration
+
+ // keepaliveDone 关闭时通知当前 keepalive goroutine 退出。
+ // 每次 finishReconnect 成功时重建,markBroken/close 时关闭。
+ keepaliveDone chan struct{}
}
// SSHDialer 提供按 domain 分组的全局 SSH 连接池。
@@ -124,6 +140,35 @@ func NewSSHDialer(cfg SSHDialerConfig) *SSHDialer {
return d
}
+// touch 刷新 lastUsed,表示该 domain 上有真实的业务数据流动。
+// 线程安全,用于 activityConn 在 Read/Write 成功时调用。
+func (entry *domainEntry) touch() {
+ entry.mu.Lock()
+ entry.lastUsed = time.Now()
+ entry.mu.Unlock()
+}
+
+type activityConn struct {
+ net.Conn
+ onActivity func()
+}
+
+func (c *activityConn) Read(b []byte) (int, error) {
+ n, err := c.Conn.Read(b)
+ if n > 0 {
+ c.onActivity()
+ }
+ return n, err
+}
+
+func (c *activityConn) Write(b []byte) (int, error) {
+ n, err := c.Conn.Write(b)
+ if n > 0 {
+ c.onActivity()
+ }
+ return n, err
+}
+
// DialContext 通过 SSH 隧道连接到目标 addr。
func (d *SSHDialer) DialContext(ctx context.Context, domainID, sshAddr, cert, key, addr string) (net.Conn, error) {
if d.closed.Load() {
@@ -139,11 +184,21 @@ func (d *SSHDialer) DialContext(ctx context.Context, domainID, sshAddr, cert, ke
conn, err := client.DialContext(ctx, "tcp", addr)
if err != nil {
if isSSHTransportError(err) {
- entry.markBroken()
+ log.GetLogger().Info("SSH channel dial failed with transport error, marking broken",
+ "domain", domainID,
+ "target", addr,
+ "err", err,
+ "errType", fmt.Sprintf("%T", err),
+ )
+ entry.markBroken("channel-dial-error")
}
return nil, fmt.Errorf("ssh proxy to %s: %w", addr, err)
}
- return conn, nil
+
+ return &activityConn{
+ Conn: conn,
+ onActivity: entry.touch,
+ }, nil
}
// Close 关闭所有 SSH 连接并停止后台 GC。
@@ -205,7 +260,6 @@ func (d *SSHDialer) getOrCreate(domainID string) *domainEntry {
// 连接借用
// ===========================================================================
-// borrow 返回可用的 SSH client。
func (entry *domainEntry) borrow(ctx context.Context, d *SSHDialer, sshAddr, cert, key string) (*ssh.Client, error) {
entry.mu.RLock()
if !entry.broken && entry.client != nil {
@@ -261,10 +315,10 @@ func (entry *domainEntry) reconnect(ctx context.Context, d *SSHDialer, sshAddr,
case <-time.After(wait):
// 退避结束,继续拨号
case <-ctx.Done():
- entry.finishReconnect(nil, ctx.Err(), d.closed.Load())
+ entry.finishReconnect(nil, ctx.Err(), d.closed.Load(), d)
return nil, ctx.Err()
case <-d.ctx.Done():
- entry.finishReconnect(nil, fmt.Errorf("ssh dialer closed"), true)
+ entry.finishReconnect(nil, fmt.Errorf("ssh dialer closed"), true, d)
return nil, fmt.Errorf("ssh dialer: closed")
}
}
@@ -272,7 +326,7 @@ func (entry *domainEntry) reconnect(ctx context.Context, d *SSHDialer, sshAddr,
// ---- 执行拨号 ----
// 合并 caller ctx 和 dialer ctx:dialer 关闭时立即取消拨号
client, err := d.dialSSHWithMergedCtx(ctx, sshAddr, cert, key)
- entry.finishReconnect(client, err, d.closed.Load())
+ entry.finishReconnect(client, err, d.closed.Load(), d)
if err != nil {
return nil, fmt.Errorf("ssh reconnect: %w", err)
}
@@ -281,7 +335,7 @@ func (entry *domainEntry) reconnect(ctx context.Context, d *SSHDialer, sshAddr,
// finishReconnect 在拨号完成后更新状态并通知等待者。
// dialerClosed 为 true 时,即使拨号成功也丢弃新连接,防止泄漏。
-func (entry *domainEntry) finishReconnect(client *ssh.Client, err error, dialerClosed bool) {
+func (entry *domainEntry) finishReconnect(client *ssh.Client, err error, dialerClosed bool, d *SSHDialer) {
entry.mu.Lock()
defer entry.mu.Unlock()
@@ -296,6 +350,15 @@ func (entry *domainEntry) finishReconnect(client *ssh.Client, err error, dialerC
entry.client = client
entry.broken = false
entry.reconnectBackoff = 0
+ // 关闭上一个 keepalive goroutine(如果有),避免泄漏
+ if entry.keepaliveDone != nil {
+ close(entry.keepaliveDone)
+ }
+ entry.keepaliveDone = make(chan struct{})
+ go entry.keepaliveLoop(client, d.cfg.KeepaliveInterval, entry.keepaliveDone)
+ if d.cfg.OnReconnect != nil {
+ d.cfg.OnReconnect(entry.domainID)
+ }
} else {
// 失败或 dialer 已关闭 → 丢弃新连接
if client != nil {
@@ -322,13 +385,70 @@ func nextBackoff(current time.Duration, max time.Duration) time.Duration {
}
// markBroken 标记连接为损坏,下次 borrow 触发重连。
-func (entry *domainEntry) markBroken() {
+func (entry *domainEntry) markBroken(reason string) {
+ entry.mu.Lock()
+ defer entry.mu.Unlock()
+ entry.markBrokenLocked(reason)
+}
+
+// markBrokenLocked 在持有 entry.mu 的情况下标记连接损坏。
+// reason 记录触发关闭的路径(cleanup/keepalive/dial-error/close),便于定位断连根因。
+func (entry *domainEntry) markBrokenLocked(reason string) {
+ if entry.client == nil {
+ return
+ }
+ log.GetLogger().Info("SSH connection marked broken",
+ "domain", entry.domainID,
+ "reason", reason,
+ "lastUsed", entry.lastUsed,
+ "idleFor", time.Since(entry.lastUsed).Round(time.Second),
+ )
+ if entry.keepaliveDone != nil {
+ close(entry.keepaliveDone)
+ entry.keepaliveDone = nil
+ }
+ entry.broken = true
+ _ = entry.client.Close()
+ entry.client = nil
+}
+
+// markBrokenIfCurrent 仅当 client 仍是当前连接时才标记损坏。
+// keepalive goroutine 检测失败时,entry 可能已被重连成新 client,
+// 防止误标新连接。
+func (entry *domainEntry) markBrokenIfCurrent(client *ssh.Client, reason string) {
entry.mu.Lock()
defer entry.mu.Unlock()
- if entry.client != nil {
- entry.broken = true
- _ = entry.client.Close()
- entry.client = nil
+ if entry.client == client && client != nil {
+ entry.markBrokenLocked(reason)
+ }
+}
+
+// keepaliveLoop 按 KeepaliveInterval 发送 SSH 应用层保活请求。
+// 任一失败(SendRequest 报错或底层连接断开)即标记 broken 并退出。
+// 通过 done channel 在连接被替换/关闭时退出,避免 goroutine 泄漏。
+func (entry *domainEntry) keepaliveLoop(client *ssh.Client, interval time.Duration, done chan struct{}) {
+ logger := log.GetLogger()
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ if _, _, err := client.SendRequest(keepaliveRequest, true, nil); err != nil {
+ // 记录具体错误类型,便于定位断连根因:
+ // - i/o timeout: 对端无响应,像会话被中间设备静默丢
+ // - connection reset by peer: 被主动 RST,像有设备踢连接
+ // - EOF: 对端正常关闭
+ logger.Info("SSH keepalive failed, marking connection broken",
+ "domain", entry.domainID,
+ "err", err,
+ "errType", fmt.Sprintf("%T", err),
+ )
+ entry.markBrokenIfCurrent(client, "keepalive-failed")
+ return
+ }
+ case <-done:
+ return
+ }
}
}
@@ -336,11 +456,7 @@ func (entry *domainEntry) markBroken() {
func (entry *domainEntry) close() {
entry.mu.Lock()
defer entry.mu.Unlock()
- if entry.client != nil {
- entry.broken = true
- _ = entry.client.Close()
- entry.client = nil
- }
+ entry.markBrokenLocked("dialer-close")
}
// ===========================================================================
@@ -373,9 +489,8 @@ func (d *SSHDialer) cleanup() {
for _, entry := range entries {
entry.mu.Lock()
if entry.client != nil && !entry.broken && entry.lastUsed.Before(cutoff) {
- entry.broken = true
- _ = entry.client.Close()
- entry.client = nil
+ // 走统一关闭逻辑,确保 keepalive goroutine 被通知退出
+ entry.markBrokenLocked("idle-cleanup")
}
entry.mu.Unlock()
}
@@ -417,7 +532,10 @@ func dialSSH(ctx context.Context, sshAddr, certPEM, keyPEM string, cfg SSHDialer
Timeout: cfg.SSHTimeout,
}
- dialer := &net.Dialer{Timeout: cfg.SSHTimeout}
+ dialer := &net.Dialer{
+ Timeout: cfg.SSHTimeout,
+ KeepAlive: 30 * time.Second,
+ }
conn, err := dialer.DialContext(ctx, "tcp", address)
if err != nil {
return nil, fmt.Errorf("dial ssh server %s: %w", address, err)
@@ -444,14 +562,31 @@ func parseSSHAddr(addr string, defaultUser string) (string, string) {
// isSSHTransportError 返回 true 当错误指示 SSH 传输层连接本身已断开,
// 而非远端目标连接失败(如 target unreachable)。
+// 调用方主动取消(context.Canceled/DeadlineExceeded)不算传输错误,
+// 避免误标健康连接。
func isSSHTransportError(err error) bool {
+ if err == nil {
+ return false
+ }
+ // 调用方主动取消不应标 broken
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ return false
+ }
// SSH 底层 TCP 断开会返回 net.OpError
var opErr *net.OpError
if errors.As(err, &opErr) {
return true
}
- // 优雅关闭返回 io.EOF
- if errors.Is(err, io.EOF) {
+ // 优雅关闭/对端中途断开
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ return true
+ }
+ // 内核层 TCP 错误
+ if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ETIMEDOUT) {
+ return true
+ }
+ // x/crypto/ssh 内部传输错误(无导出 sentinel)
+ if strings.Contains(err.Error(), "ssh: tcp transport closed") {
return true
}
return false
diff --git a/apps/rlark/pkg/agent/container/ssh_dialer_test.go b/apps/rlark/pkg/agent/container/ssh_dialer_test.go
index 8ec96e5..75091f6 100644
--- a/apps/rlark/pkg/agent/container/ssh_dialer_test.go
+++ b/apps/rlark/pkg/agent/container/ssh_dialer_test.go
@@ -2,8 +2,12 @@ package container
import (
"context"
+ "errors"
+ "fmt"
+ "io"
"net"
"sync"
+ "syscall"
"testing"
"time"
@@ -137,7 +141,7 @@ func TestDomainEntry_MarkBroken(t *testing.T) {
client := newSSHClient(t)
entry.client = client
- entry.markBroken()
+ entry.markBroken("test")
if !entry.broken {
t.Fatal("expected broken=true")
}
@@ -180,7 +184,7 @@ func TestSSHDialer_ConcurrentReconnect(t *testing.T) {
entry := d.getOrCreate("test-domain")
// 模拟连接断开
- entry.markBroken()
+ entry.markBroken("test")
// 50 个并发请求,全部尝试重连(预期失败,但无惊群)
var wg sync.WaitGroup
@@ -327,7 +331,7 @@ func TestSSHDialer_Stats(t *testing.T) {
t.Fatalf("expected 2 open, got %d", stats)
}
- entry1.markBroken()
+ entry1.markBroken("test")
if stats := d.Stats(); stats != 1 {
t.Fatalf("expected 1 open after broken, got %d", stats)
}
@@ -462,12 +466,13 @@ func waitCh() <-chan struct{} {
// TestSSHDialer_BackoffReset 测试重连成功后退避重置。
func TestSSHDialer_BackoffReset(t *testing.T) {
+ d := testDialer(t)
entry := &domainEntry{domainID: "test", maxBackoff: maxReconnectBackoff}
entry.reconnectCh = make(chan struct{})
entry.reconnectBackoff = 10 * time.Second
// 模拟成功
- entry.finishReconnect(newSSHClient(t), nil, false)
+ entry.finishReconnect(newSSHClient(t), nil, false, d)
if entry.reconnectBackoff != 0 {
t.Fatalf("expected backoff reset to 0, got %v", entry.reconnectBackoff)
}
@@ -475,7 +480,7 @@ func TestSSHDialer_BackoffReset(t *testing.T) {
entry.reconnectCh = make(chan struct{})
// 模拟失败
- entry.finishReconnect(nil, assertAnError("fail"), false)
+ entry.finishReconnect(nil, assertAnError("fail"), false, d)
if entry.reconnectBackoff != initialReconnectBackoff {
t.Fatalf("expected backoff %v, got %v", initialReconnectBackoff, entry.reconnectBackoff)
}
@@ -483,7 +488,7 @@ func TestSSHDialer_BackoffReset(t *testing.T) {
entry.reconnectCh = make(chan struct{})
// 模拟再次失败
- entry.finishReconnect(nil, assertAnError("fail again"), false)
+ entry.finishReconnect(nil, assertAnError("fail again"), false, d)
expected := initialReconnectBackoff * 2
if entry.reconnectBackoff != expected {
t.Fatalf("expected backoff %v, got %v", expected, entry.reconnectBackoff)
@@ -520,6 +525,68 @@ func TestNextBackoff(t *testing.T) {
}
}
+// TestIsSSHTransportError 覆盖各类传输错误与误判场景。
+func TestIsSSHTransportError(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ want bool
+ }{
+ {"nil", nil, false},
+ {"io.EOF", io.EOF, true},
+ {"io.ErrUnexpectedEOF", io.ErrUnexpectedEOF, true},
+ {"ECONNRESET", syscall.ECONNRESET, true},
+ {"EPIPE", syscall.EPIPE, true},
+ {"ETIMEDOUT", syscall.ETIMEDOUT, true},
+ {"wrapped ECONNRESET", fmt.Errorf("dial: %w", syscall.ECONNRESET), true},
+ {"net.OpError", &net.OpError{Op: "read", Err: syscall.ECONNRESET}, true},
+ {"wrapped net.OpError", fmt.Errorf("proxy: %w", &net.OpError{Op: "read", Err: syscall.ECONNRESET}), true},
+ {"ssh transport closed", errors.New("ssh: tcp transport closed"), true},
+ {"context.Canceled", context.Canceled, false},
+ {"context.DeadlineExceeded", context.DeadlineExceeded, false},
+ {"wrapped context.Canceled", fmt.Errorf("dial: %w", context.Canceled), false},
+ {"generic error", errors.New("connection refused"), false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := isSSHTransportError(tt.err); got != tt.want {
+ t.Errorf("isSSHTransportError(%v) = %v, want %v", tt.err, got, tt.want)
+ }
+ })
+ }
+}
+
+// TestDomainEntry_MarkBrokenIfCurrent 防止 keepalive goroutine 误标新连接。
+func TestDomainEntry_MarkBrokenIfCurrent(t *testing.T) {
+ entry := &domainEntry{domainID: "test"}
+
+ old := newSSHClient(t)
+ entry.client = old
+ entry.broken = false
+
+ // 换一个"新"client 进来(模拟重连成功)
+ newClient := newSSHClient(t)
+ entry.client = newClient
+
+ // keepalive 拿旧 client 来标 broken,不应影响新连接
+ entry.markBrokenIfCurrent(old, "test")
+ if entry.broken {
+ t.Fatal("should not mark broken when client has been replaced")
+ }
+ if entry.client != newClient {
+ t.Fatal("current client should remain untouched")
+ }
+
+ // 用当前 client 标 broken,应生效
+ entry.markBrokenIfCurrent(newClient, "test")
+ if !entry.broken {
+ t.Fatal("expected broken=true when marking current client")
+ }
+ if entry.client != nil {
+ t.Fatal("expected client to be nil after markBrokenIfCurrent")
+ }
+}
+
// TestParseSSHAddr 测试 user@host:port 解析。
func TestParseSSHAddr(t *testing.T) {
tests := []struct {
diff --git a/apps/rlark/pkg/agent/controllers/base/controller.go b/apps/rlark/pkg/agent/controllers/base/controller.go
index 9565189..97388a7 100644
--- a/apps/rlark/pkg/agent/controllers/base/controller.go
+++ b/apps/rlark/pkg/agent/controllers/base/controller.go
@@ -1,12 +1,100 @@
package base
import (
+ "context"
"fmt"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/predicate"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
)
+// managementTaskNameAnnotation marks K8s workloads (and their pods) that back a
+// management Task. It mirrors task.ManagementTaskNameAnnotation; duplicated here
+// to avoid an import cycle (the task package imports base).
+const managementTaskNameAnnotation = "rlark.io/management-task-name"
+
+// podOwningKind returns the Kubernetes Kind of pod-owning workloads
+// (Deployment/StatefulSet/DaemonSet) for the given push-reconciler object type,
+// or "" if the object is not a pod-owning workload.
+func podOwningKind(obj client.Object) string {
+ switch obj.(type) {
+ case *appsv1.Deployment:
+ return "Deployment"
+ case *appsv1.StatefulSet:
+ return "StatefulSet"
+ case *appsv1.DaemonSet:
+ return "DaemonSet"
+ }
+ return ""
+}
+
+// enqueueOwningWorkload maps a Pod change to the reconcile request of the
+// workload that owns it (Deployment/StatefulSet/DaemonSet). StatefulSet and
+// DaemonSet pods reference their owning workload directly via OwnerReferences;
+// Deployment pods are owned by a ReplicaSet which is in turn owned by the
+// Deployment, so the ReplicaSet is resolved to find the Deployment.
+//
+// Pod container status transitions (e.g. a container entering CrashLoopBackOff)
+// are not reflected in the workload's own status fields (restart counts and
+// waiting reasons live on the Pod, not on the StatefulSet/Deployment status).
+// Without watching Pods, a workload whose status has stabilized — e.g. a
+// StatefulSet whose readyReplicas never reached desired because its pod crashed
+// on the first start — would never be re-reconciled, so the failure stays
+// hidden and the Task status is never updated.
+func enqueueOwningWorkload(localClient client.Client, wantKind string) handler.EventHandler {
+ return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+ pod, ok := obj.(*corev1.Pod)
+ if !ok {
+ return nil
+ }
+ return podOwnerRequests(ctx, localClient, pod, wantKind)
+ })
+}
+
+// podOwnerRequests resolves the workload (of kind wantKind) that owns pod and
+// returns the reconcile request for it. StatefulSet/DaemonSet pods reference
+// their owning workload directly; Deployment pods are owned by a ReplicaSet
+// which is resolved to find the owning Deployment.
+func podOwnerRequests(ctx context.Context, localClient client.Client, pod *corev1.Pod, wantKind string) []reconcile.Request {
+ owner := metav1.GetControllerOf(pod)
+ if owner == nil {
+ return nil
+ }
+ // Direct ownership (StatefulSet/DaemonSet pods).
+ if owner.Kind == wantKind {
+ return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: owner.Name, Namespace: pod.Namespace}}}
+ }
+ // Indirect ownership: Pod -> ReplicaSet -> Deployment.
+ if owner.Kind == "ReplicaSet" && wantKind == "Deployment" {
+ var rs appsv1.ReplicaSet
+ if err := localClient.Get(ctx, types.NamespacedName{Name: owner.Name, Namespace: pod.Namespace}, &rs); err != nil {
+ return nil
+ }
+ if depOwner := metav1.GetControllerOf(&rs); depOwner != nil && depOwner.Kind == wantKind {
+ return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: depOwner.Name, Namespace: rs.Namespace}}}
+ }
+ }
+ return nil
+}
+
+// hasManagementTaskAnnotation is a predicate that only lets through K8s objects
+// carrying the rlark management-task annotation. It restricts the Pod watch to
+// rlark task pods so non-rlark pods in the cluster don't trigger (potentially
+// costly) reconciliation or ReplicaSet lookups.
+func hasManagementTaskAnnotation() predicate.Predicate {
+ return predicate.NewPredicateFuncs(func(obj client.Object) bool {
+ return obj.GetAnnotations()[managementTaskNameAnnotation] != ""
+ })
+}
+
// Reconciler reconciles resources.
type Reconciler interface {
KubernetesResource() KubernetesResource
@@ -56,11 +144,26 @@ func (c *Controller) SetupPushController(mgr any) error {
return fmt.Errorf("controller is not running in a Kubernetes environment")
}
for kubeResource, reconciler := range c.C.AsKubePushReconcilers() {
- err := ctrl.NewControllerManagedBy(kubeMgr).
+ blder := ctrl.NewControllerManagedBy(kubeMgr).
For(kubeResource.Type).
- Named(kubeResource.Name + "-push").
- Complete(reconciler)
- if err != nil {
+ Named(kubeResource.Name + "-push")
+
+ // For pod-owning workloads (Deployment/StatefulSet/DaemonSet),
+ // also watch Pods so that container status changes — most
+ // importantly a container entering CrashLoopBackOff — re-trigger
+ // reconciliation. The workload's own status does not reflect
+ // per-container waiting/restart state, so without this a pod that
+ // crashes after the workload status stabilized is never detected
+ // and the Task keeps reporting a stale phase.
+ if kind := podOwningKind(kubeResource.Type); kind != "" {
+ blder = blder.Watches(
+ &corev1.Pod{},
+ enqueueOwningWorkload(c.LocalKubeClient, kind),
+ builder.WithPredicates(hasManagementTaskAnnotation()),
+ )
+ }
+
+ if err := blder.Complete(reconciler); err != nil {
return fmt.Errorf("failed to setup push controller for %s: %w", kubeResource.Name, err)
}
}
diff --git a/apps/rlark/pkg/agent/controllers/base/controller_test.go b/apps/rlark/pkg/agent/controllers/base/controller_test.go
new file mode 100644
index 0000000..b9fe92f
--- /dev/null
+++ b/apps/rlark/pkg/agent/controllers/base/controller_test.go
@@ -0,0 +1,82 @@
+package base
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func controllerRef(kind, name string) metav1.OwnerReference {
+ t := true
+ return metav1.OwnerReference{Kind: kind, Name: name, Controller: &t, APIVersion: "apps/v1"}
+}
+
+func makePod(name, ns string, owners ...metav1.OwnerReference) *corev1.Pod {
+ return &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: ns,
+ OwnerReferences: owners,
+ Annotations: map[string]string{managementTaskNameAnnotation: "some-task"},
+ },
+ }
+}
+
+func TestPodOwnerRequests(t *testing.T) {
+ ctx := context.Background()
+
+ // StatefulSet pod: owned directly by the StatefulSet.
+ stsPod := makePod("actor-0", "rlark-system", controllerRef("StatefulSet", "robot-policy-training-actor"))
+ got := podOwnerRequests(ctx, fake.NewClientBuilder().Build(), stsPod, "StatefulSet")
+ assert.Equal(t, []reconcile.Request{
+ {NamespacedName: types.NamespacedName{Name: "robot-policy-training-actor", Namespace: "rlark-system"}},
+ }, got)
+
+ // DaemonSet pod: owned directly by the DaemonSet.
+ dsPod := makePod("ds-pod", "rlark-system", controllerRef("DaemonSet", "my-daemonset"))
+ got = podOwnerRequests(ctx, fake.NewClientBuilder().Build(), dsPod, "DaemonSet")
+ assert.Equal(t, []reconcile.Request{
+ {NamespacedName: types.NamespacedName{Name: "my-daemonset", Namespace: "rlark-system"}},
+ }, got)
+
+ // Deployment pod: owned by a ReplicaSet that is owned by the Deployment.
+ rs := &appsv1.ReplicaSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "app-abc",
+ Namespace: "rlark-system",
+ OwnerReferences: []metav1.OwnerReference{controllerRef("Deployment", "my-app")},
+ },
+ }
+ depPod := makePod("app-xyz", "rlark-system", controllerRef("ReplicaSet", "app-abc"))
+ got = podOwnerRequests(ctx, fake.NewClientBuilder().WithObjects(rs).Build(), depPod, "Deployment")
+ assert.Equal(t, []reconcile.Request{
+ {NamespacedName: types.NamespacedName{Name: "my-app", Namespace: "rlark-system"}},
+ }, got)
+
+ // A StatefulSet push controller must not enqueue for a Deployment-owned pod.
+ got = podOwnerRequests(ctx, fake.NewClientBuilder().WithObjects(rs).Build(), depPod, "StatefulSet")
+ assert.Empty(t, got)
+
+ // Pod without a controller owner yields nothing.
+ noOwnerPod := makePod("standalone", "rlark-system")
+ assert.Empty(t, podOwnerRequests(ctx, fake.NewClientBuilder().Build(), noOwnerPod, "StatefulSet"))
+
+ // Deployment pod whose ReplicaSet has already been deleted yields nothing.
+ orphanPod := makePod("app-gone", "rlark-system", controllerRef("ReplicaSet", "missing-rs"))
+ assert.Empty(t, podOwnerRequests(ctx, fake.NewClientBuilder().Build(), orphanPod, "Deployment"))
+}
+
+func TestPodOwningKind(t *testing.T) {
+ assert.Equal(t, "Deployment", podOwningKind(&appsv1.Deployment{}))
+ assert.Equal(t, "StatefulSet", podOwningKind(&appsv1.StatefulSet{}))
+ assert.Equal(t, "DaemonSet", podOwningKind(&appsv1.DaemonSet{}))
+ assert.Equal(t, "", podOwningKind(&corev1.Pod{}))
+ assert.Equal(t, "", podOwningKind(&corev1.Node{}))
+}
diff --git a/apps/rlark/pkg/agent/controllers/pod/push.go b/apps/rlark/pkg/agent/controllers/pod/push.go
index 9e97523..d00dd41 100644
--- a/apps/rlark/pkg/agent/controllers/pod/push.go
+++ b/apps/rlark/pkg/agent/controllers/pod/push.go
@@ -50,6 +50,16 @@ func (r *pushPodReconciler) Reconcile(ctx context.Context, req reconcile.Request
func (r *pushPodReconciler) buildRLarkPodFromK8sPod(k8sPod *corev1.Pod, taskName, taskNamespace string) *rlarkv1alpha1.Pod {
phase := convertK8sPodPhase(k8sPod.Status.Phase)
+ message := k8sPod.Status.Message
+ // A K8s Pod can be phase=Running while its main container (the container
+ // named "main") is stuck in CrashLoopBackOff. Surface that as Failed on
+ // the management Pod CR so operators see the failure immediately instead
+ // of a misleading Running/Pending, and the UI tooltip can show the
+ // waiting message. Only override when the pod has not already Succeeded.
+ if clMsg, crashed := mainContainerCrashLoopMessage(k8sPod); crashed && phase != rlarkv1alpha1.PodPhaseSucceeded {
+ phase = rlarkv1alpha1.PodPhaseFailed
+ message = clMsg
+ }
podSpec := rlarkv1alpha1.PodSpec{
TaskNamespace: taskNamespace,
@@ -66,7 +76,7 @@ func (r *pushPodReconciler) buildRLarkPodFromK8sPod(k8sPod *corev1.Pod, taskName
Phase: phase,
Node: k8sPod.Spec.NodeName,
IP: k8sPod.Status.PodIP,
- Message: k8sPod.Status.Message,
+ Message: message,
}
// Labels enable lookup by k8s pod name/namespace (e.g. for deletion when only the
@@ -169,3 +179,25 @@ func convertK8sPodPhase(phase corev1.PodPhase) rlarkv1alpha1.PodPhase {
return rlarkv1alpha1.PodPhasePending
}
}
+
+// mainContainerCrashLoopMessage inspects the workload's main container (the
+// container named "main") of a K8s Pod. When it is in the CrashLoopBackOff
+// waiting state, it returns the waiting message (falling back to the reason)
+// and true so the management Pod CR can be surfaced as Failed. This augments
+// the raw pod phase: a Pod can be phase=Running while its main container is
+// stuck in CrashLoopBackOff.
+func mainContainerCrashLoopMessage(k8sPod *corev1.Pod) (string, bool) {
+ for _, cs := range k8sPod.Status.ContainerStatuses {
+ if cs.Name != "main" {
+ continue
+ }
+ if waiting := cs.State.Waiting; waiting != nil && waiting.Reason == "CrashLoopBackOff" {
+ msg := waiting.Message
+ if msg == "" {
+ msg = waiting.Reason
+ }
+ return msg, true
+ }
+ }
+ return "", false
+}
diff --git a/apps/rlark/pkg/agent/controllers/task/pull.go b/apps/rlark/pkg/agent/controllers/task/pull.go
index 38b0782..e7d6685 100644
--- a/apps/rlark/pkg/agent/controllers/task/pull.go
+++ b/apps/rlark/pkg/agent/controllers/task/pull.go
@@ -5,6 +5,7 @@ import (
"fmt"
"slices"
"strings"
+ "time"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -36,6 +37,9 @@ const (
PVCTaskLabel = "rlark.io/task"
PVCOwnerAnnotation = "rlark.io/pvc-owner"
PVCOwnerTaskAnnotation = "rlark.io/pvc-owner-task"
+ RestartedAtAnnotation = "rlark.io/restarted-at"
+ StoppedAnnotation = "rlark.io/stopped"
+ CleanupRequeueInterval = 2 * time.Second
)
// pullReconciler watches management Tasks and creates workloads on local cluster.
@@ -51,8 +55,13 @@ func (r *pullReconciler) Reconcile(ctx context.Context, req reconcile.Request) (
if err := r.c.ManagementClient.Get(ctx, req.NamespacedName, &mgmtTask); err != nil {
if client.IgnoreNotFound(err) == nil {
logger.Info("management Task deleted, cleaning up local workload")
- if cleanupErr := r.cleanupWorkload(ctx, req.Name, req.Namespace); cleanupErr != nil {
+ pending, cleanupErr := r.cleanupWorkload(ctx, req.Name, "rlark-system")
+ if cleanupErr != nil {
logger.Error(cleanupErr, "failed to clean up workload")
+ return reconcile.Result{}, cleanupErr
+ }
+ if pending {
+ return reconcile.Result{RequeueAfter: CleanupRequeueInterval}, nil
}
return reconcile.Result{}, nil
}
@@ -63,10 +72,14 @@ func (r *pullReconciler) Reconcile(ctx context.Context, req reconcile.Request) (
if mgmtTask.DeletionTimestamp != nil {
logger.Info("management Task being deleted, cleaning up local workload")
workloadNs := getWorkloadNamespace(&mgmtTask)
- if err := r.cleanupWorkload(ctx, mgmtTask.Name, workloadNs); err != nil {
+ pending, err := r.cleanupWorkload(ctx, mgmtTask.Name, workloadNs)
+ if err != nil {
logger.Error(err, "failed to clean up workload")
return reconcile.Result{}, err
}
+ if pending {
+ return reconcile.Result{RequeueAfter: CleanupRequeueInterval}, nil
+ }
mgmtTask.Finalizers = slices.DeleteFunc(mgmtTask.Finalizers, func(s string) bool {
return s == ManagementTaskFinalizer
})
@@ -99,11 +112,33 @@ func (r *pullReconciler) Reconcile(ctx context.Context, req reconcile.Request) (
}
workloadSpec := mgmtTask.Spec.Kubernetes.Workload
+ workloadNamespace := getWorkloadNamespace(&mgmtTask)
+ restarting, err := r.restartCleanupRequired(ctx, &mgmtTask, workloadNamespace)
+ if err != nil {
+ return reconcile.Result{}, err
+ }
+ if mgmtTask.Annotations[StoppedAnnotation] != "true" &&
+ (restarting || mgmtTask.Status.Phase == rlarkv1alpha1.TaskPhaseStopped) &&
+ mgmtTask.Status.Phase != rlarkv1alpha1.TaskPhasePending {
+ return updateMgmtTaskStatus(ctx, logger, r.c.ManagementClient, &mgmtTask, rlarkv1alpha1.TaskPhasePending, "", nil)
+ }
+ if mgmtTask.Annotations[StoppedAnnotation] == "true" || restarting {
+ pending, err := r.cleanupWorkload(ctx, mgmtTask.Name, workloadNamespace)
+ if err != nil {
+ return reconcile.Result{}, err
+ }
+ if pending {
+ return reconcile.Result{RequeueAfter: CleanupRequeueInterval}, nil
+ }
+ if mgmtTask.Annotations[StoppedAnnotation] == "true" {
+ return updateMgmtTaskStatus(ctx, logger, r.c.ManagementClient, &mgmtTask, rlarkv1alpha1.TaskPhaseStopped, "", nil)
+ }
+ }
+
applyTemplateMutations(&workloadSpec.Template, &mgmtTask, r.c.Image)
- workloadNamespace := getWorkloadNamespace(&mgmtTask)
if err := r.ensureImagePullSecrets(ctx, &workloadSpec.Template, workloadNamespace); err != nil {
- logger.Error(err, "failed to ensure image pull secrets")
+ return reconcile.Result{}, fmt.Errorf("ensure image pull secrets: %w", err)
}
if err := r.ensurePVCs(ctx, &mgmtTask, workloadSpec); err != nil {
@@ -170,15 +205,36 @@ func (r *pullReconciler) createOrUpdateWorkload(
return reconcile.Result{}, nil
}
+func (r *pullReconciler) restartCleanupRequired(ctx context.Context, mgmtTask *rlarkv1alpha1.Task, namespace string) (bool, error) {
+ restartedAt := mgmtTask.Annotations[RestartedAtAnnotation]
+ if restartedAt == "" {
+ return false, nil
+ }
+ key := types.NamespacedName{Name: mgmtTask.Name, Namespace: namespace}
+ for _, obj := range []client.Object{&appsv1.Deployment{}, &appsv1.DaemonSet{}, &appsv1.StatefulSet{}} {
+ err := r.c.LocalKubeClient.Get(ctx, key, obj)
+ if err == nil {
+ return obj.GetAnnotations()[RestartedAtAnnotation] != restartedAt, nil
+ }
+ if !errors.IsNotFound(err) {
+ return false, err
+ }
+ }
+
+ var pvcs corev1.PersistentVolumeClaimList
+ if err := r.c.LocalKubeClient.List(ctx, &pvcs, client.InNamespace(namespace), client.MatchingLabels{PVCTaskLabel: mgmtTask.Name}); err != nil {
+ return false, err
+ }
+ return len(pvcs.Items) > 0, nil
+}
+
func (r *pullReconciler) createOrUpdateDeployment(ctx context.Context, mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.KubernetesWorkloadSpec) (reconcile.Result, error) {
return r.createOrUpdateWorkload(ctx, mgmtTask, "Deployment",
&appsv1.Deployment{},
buildDeployment(mgmtTask, spec),
func(obj client.Object) {
deploy := obj.(*appsv1.Deployment)
- if deploy.Annotations == nil {
- deploy.Annotations = make(map[string]string)
- }
+ deploy.Annotations = workloadAnnotations(mgmtTask)
deploy.Annotations[ManagementTaskResourceVersionAnnotation] = mgmtTask.ResourceVersion
deploy.Spec.Replicas = spec.Replicas
deploy.Spec.Template = spec.Template
@@ -191,9 +247,7 @@ func (r *pullReconciler) createOrUpdateDaemonSet(ctx context.Context, mgmtTask *
buildDaemonSet(mgmtTask, spec),
func(obj client.Object) {
ds := obj.(*appsv1.DaemonSet)
- if ds.Annotations == nil {
- ds.Annotations = make(map[string]string)
- }
+ ds.Annotations = workloadAnnotations(mgmtTask)
ds.Annotations[ManagementTaskResourceVersionAnnotation] = mgmtTask.ResourceVersion
ds.Spec.Template = spec.Template
})
@@ -205,42 +259,50 @@ func (r *pullReconciler) createOrUpdateStatefulSet(ctx context.Context, mgmtTask
buildStatefulSet(mgmtTask, spec),
func(obj client.Object) {
sts := obj.(*appsv1.StatefulSet)
- if sts.Annotations == nil {
- sts.Annotations = make(map[string]string)
- }
+ sts.Annotations = workloadAnnotations(mgmtTask)
sts.Annotations[ManagementTaskResourceVersionAnnotation] = mgmtTask.ResourceVersion
sts.Spec.Replicas = spec.Replicas
sts.Spec.Template = spec.Template
})
}
-func (r *pullReconciler) cleanupWorkload(ctx context.Context, name string, namespace string) error {
+func (r *pullReconciler) cleanupWorkload(ctx context.Context, name string, namespace string) (bool, error) {
if r.c.LocalKubeClient == nil {
- return nil
+ return false, nil
}
+ pending := false
workloadKey := types.NamespacedName{Name: name, Namespace: namespace}
for _, obj := range []client.Object{&appsv1.Deployment{}, &appsv1.DaemonSet{}, &appsv1.StatefulSet{}} {
- if err := r.c.LocalKubeClient.Get(ctx, workloadKey, obj); err == nil {
- if err := r.c.LocalKubeClient.Delete(ctx, obj); err != nil {
- return err
+ err := r.c.LocalKubeClient.Get(ctx, workloadKey, obj)
+ if err != nil && !errors.IsNotFound(err) {
+ return false, err
+ }
+ if err == nil {
+ pending = true
+ if obj.GetDeletionTimestamp().IsZero() {
+ if err := r.c.LocalKubeClient.Delete(ctx, obj); err != nil && !errors.IsNotFound(err) {
+ return false, err
+ }
}
}
}
svcKey := types.NamespacedName{Name: rayHeadServiceName(name), Namespace: namespace}
var svc corev1.Service
- if err := r.c.LocalKubeClient.Get(ctx, svcKey, &svc); err == nil {
- if err := r.c.LocalKubeClient.Delete(ctx, &svc); err != nil {
- return err
+ if err := r.c.LocalKubeClient.Get(ctx, svcKey, &svc); err == nil && svc.DeletionTimestamp.IsZero() {
+ if err := r.c.LocalKubeClient.Delete(ctx, &svc); err != nil && !errors.IsNotFound(err) {
+ return false, err
}
+ } else if err != nil && !errors.IsNotFound(err) {
+ return false, err
}
- if err := r.cleanupPVCs(ctx, name, namespace); err != nil {
- return err
+ pvcsPending, err := r.cleanupPVCs(ctx, name, namespace)
+ if err != nil {
+ return false, err
}
-
- return nil
+ return pending || pvcsPending, nil
}
func (r *pullReconciler) ensureRayResources(ctx context.Context, mgmtTask *rlarkv1alpha1.Task, owner client.Object) error {
@@ -374,9 +436,9 @@ func (r *pullReconciler) ensurePVCs(ctx context.Context, mgmtTask *rlarkv1alpha1
return nil
}
-func (r *pullReconciler) cleanupPVCs(ctx context.Context, taskName string, namespace string) error {
+func (r *pullReconciler) cleanupPVCs(ctx context.Context, taskName string, namespace string) (bool, error) {
if r.c.LocalKubeClient == nil {
- return nil
+ return false, nil
}
pvcList := &corev1.PersistentVolumeClaimList{}
@@ -384,18 +446,18 @@ func (r *pullReconciler) cleanupPVCs(ctx context.Context, taskName string, names
PVCTaskLabel: taskName,
}); err != nil {
if errors.IsNotFound(err) {
- return nil
+ return false, nil
}
- return fmt.Errorf("list PVCs for task %s: %w", taskName, err)
+ return false, fmt.Errorf("list PVCs for task %s: %w", taskName, err)
}
if len(pvcList.Items) == 0 {
var allPVCs corev1.PersistentVolumeClaimList
if err := r.c.LocalKubeClient.List(ctx, &allPVCs, client.InNamespace(namespace)); err != nil {
if errors.IsNotFound(err) {
- return nil
+ return false, nil
}
- return fmt.Errorf("list all PVCs in namespace %s: %w", namespace, err)
+ return false, fmt.Errorf("list all PVCs in namespace %s: %w", namespace, err)
}
for _, pvc := range allPVCs.Items {
if pvc.Annotations != nil && pvc.Annotations[PVCOwnerTaskAnnotation] == taskName {
@@ -407,14 +469,14 @@ func (r *pullReconciler) cleanupPVCs(ctx context.Context, taskName string, names
for i := range pvcList.Items {
logger := log.FromContext(ctx).WithValues("pvc", pvcList.Items[i].Name)
logger.Info("Deleting PVC owned by task", "pvc", pvcList.Items[i].Name, "task", taskName)
- if err := r.c.LocalKubeClient.Delete(ctx, &pvcList.Items[i]); err != nil {
- if !errors.IsNotFound(err) {
- return fmt.Errorf("delete PVC %s: %w", pvcList.Items[i].Name, err)
+ if pvcList.Items[i].DeletionTimestamp.IsZero() {
+ if err := r.c.LocalKubeClient.Delete(ctx, &pvcList.Items[i]); err != nil && !errors.IsNotFound(err) {
+ return false, fmt.Errorf("delete PVC %s: %w", pvcList.Items[i].Name, err)
}
}
}
- return nil
+ return len(pvcList.Items) > 0, nil
}
func pvcNameForVolume(taskName, volumeName string) string {
@@ -428,6 +490,19 @@ func getWorkloadNamespace(mgmtTask *rlarkv1alpha1.Task) string {
// --- workload builder functions ---
+func workloadAnnotations(mgmtTask *rlarkv1alpha1.Task) map[string]string {
+ annotations := map[string]string{
+ ManagementTaskNameAnnotation: mgmtTask.Name,
+ ManagementTaskNamespaceAnnotation: mgmtTask.Namespace,
+ ManagementTaskUIDAnnotation: string(mgmtTask.UID),
+ ManagementTaskResourceVersionAnnotation: mgmtTask.ResourceVersion,
+ }
+ if restartedAt := mgmtTask.Annotations[RestartedAtAnnotation]; restartedAt != "" {
+ annotations[RestartedAtAnnotation] = restartedAt
+ }
+ return annotations
+}
+
// ensureLabels ensures the pod template has labels, adding a default if none are set.
func ensureLabels(template *corev1.PodTemplateSpec, name string) {
if template == nil || len(template.Labels) == 0 {
@@ -534,7 +609,7 @@ func (r *pullReconciler) ensureImagePullSecrets(ctx context.Context, template *c
// Build a map of registry prefix -> secret name
registryToSecret := make(map[string]string, len(secretList.Items))
for _, secret := range secretList.Items {
- registry := secret.Annotations[common.ImageRegistryAnnotationRegistry]
+ registry := common.NormalizeRegistry(secret.Annotations[common.ImageRegistryAnnotationRegistry])
if registry == "" {
continue
}
@@ -544,6 +619,7 @@ func (r *pullReconciler) ensureImagePullSecrets(ctx context.Context, template *c
// Find matching registries for our images
matchedSecrets := make(map[string]bool)
for _, image := range imageRefs {
+ image = common.NormalizeRegistry(image)
for registry, secretName := range registryToSecret {
if strings.HasPrefix(image, registry+"/") || image == registry {
matchedSecrets[secretName] = true
@@ -560,7 +636,7 @@ func (r *pullReconciler) ensureImagePullSecrets(ctx context.Context, template *c
for secretName := range matchedSecrets {
if err := r.syncImagePullSecret(ctx, secretName, workloadNamespace); err != nil {
logger.Error(err, "failed to sync image pull secret", "secret", secretName, "namespace", workloadNamespace)
- continue
+ return fmt.Errorf("sync image pull secret %q: %w", secretName, err)
}
}
@@ -618,14 +694,9 @@ func (r *pullReconciler) syncImagePullSecret(ctx context.Context, secretName, de
func buildDeployment(mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.KubernetesWorkloadSpec) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
- Name: mgmtTask.Name,
- Namespace: getWorkloadNamespace(mgmtTask),
- Annotations: map[string]string{
- ManagementTaskNameAnnotation: mgmtTask.Name,
- ManagementTaskNamespaceAnnotation: mgmtTask.Namespace,
- ManagementTaskUIDAnnotation: string(mgmtTask.UID),
- ManagementTaskResourceVersionAnnotation: mgmtTask.ResourceVersion,
- },
+ Name: mgmtTask.Name,
+ Namespace: getWorkloadNamespace(mgmtTask),
+ Annotations: workloadAnnotations(mgmtTask),
},
Spec: appsv1.DeploymentSpec{
Replicas: spec.Replicas,
@@ -640,14 +711,9 @@ func buildDeployment(mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.Kubernete
func buildDaemonSet(mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.KubernetesWorkloadSpec) *appsv1.DaemonSet {
return &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
- Name: mgmtTask.Name,
- Namespace: getWorkloadNamespace(mgmtTask),
- Annotations: map[string]string{
- ManagementTaskNameAnnotation: mgmtTask.Name,
- ManagementTaskNamespaceAnnotation: mgmtTask.Namespace,
- ManagementTaskUIDAnnotation: string(mgmtTask.UID),
- ManagementTaskResourceVersionAnnotation: mgmtTask.ResourceVersion,
- },
+ Name: mgmtTask.Name,
+ Namespace: getWorkloadNamespace(mgmtTask),
+ Annotations: workloadAnnotations(mgmtTask),
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
@@ -661,14 +727,9 @@ func buildDaemonSet(mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.Kubernetes
func buildStatefulSet(mgmtTask *rlarkv1alpha1.Task, spec *rlarkv1alpha1.KubernetesWorkloadSpec) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
- Name: mgmtTask.Name,
- Namespace: getWorkloadNamespace(mgmtTask),
- Annotations: map[string]string{
- ManagementTaskNameAnnotation: mgmtTask.Name,
- ManagementTaskNamespaceAnnotation: mgmtTask.Namespace,
- ManagementTaskUIDAnnotation: string(mgmtTask.UID),
- ManagementTaskResourceVersionAnnotation: mgmtTask.ResourceVersion,
- },
+ Name: mgmtTask.Name,
+ Namespace: getWorkloadNamespace(mgmtTask),
+ Annotations: workloadAnnotations(mgmtTask),
},
Spec: appsv1.StatefulSetSpec{
Replicas: spec.Replicas,
diff --git a/apps/rlark/pkg/agent/controllers/task/pull_test.go b/apps/rlark/pkg/agent/controllers/task/pull_test.go
new file mode 100644
index 0000000..2378ace
--- /dev/null
+++ b/apps/rlark/pkg/agent/controllers/task/pull_test.go
@@ -0,0 +1,94 @@
+package task
+
+import (
+ "context"
+ "testing"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+ rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1"
+ "github.com/rlinf/rlark/apps/rlark/pkg/agent/controllers/base"
+)
+
+func TestCleanupWorkloadWaitsForStatefulSetAndPVC(t *testing.T) {
+ scheme := runtime.NewScheme()
+ if err := appsv1.AddToScheme(scheme); err != nil {
+ t.Fatal(err)
+ }
+ if err := corev1.AddToScheme(scheme); err != nil {
+ t.Fatal(err)
+ }
+
+ sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{
+ Name: "task", Namespace: "rlark-system", Finalizers: []string{"test/finalizer"},
+ }}
+ pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
+ Name: "data", Namespace: "rlark-system", Finalizers: []string{"test/finalizer"},
+ Labels: map[string]string{PVCTaskLabel: "task"},
+ }}
+ localClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sts, pvc).Build()
+ r := &pullReconciler{c: NewTaskController(base.Controller{LocalKubeClient: localClient})}
+
+ pending, err := r.cleanupWorkload(context.Background(), "task", "rlark-system")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !pending {
+ t.Fatal("cleanup should remain pending while StatefulSet/PVC still exist")
+ }
+
+ var remainingSTS appsv1.StatefulSet
+ if err := localClient.Get(context.Background(), types.NamespacedName{Name: "task", Namespace: "rlark-system"}, &remainingSTS); err != nil {
+ t.Fatalf("StatefulSet should still exist until its finalizer is cleared: %v", err)
+ }
+ var remainingPVC corev1.PersistentVolumeClaim
+ if err := localClient.Get(context.Background(), types.NamespacedName{Name: "data", Namespace: "rlark-system"}, &remainingPVC); err != nil {
+ t.Fatalf("PVC should still exist until its finalizer is cleared: %v", err)
+ }
+}
+
+func TestRestartCleanupRequiredAndAnnotationPropagation(t *testing.T) {
+ scheme := runtime.NewScheme()
+ if err := appsv1.AddToScheme(scheme); err != nil {
+ t.Fatal(err)
+ }
+ oldSTS := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{
+ Name: "task", Namespace: "rlark-system",
+ Annotations: map[string]string{RestartedAtAnnotation: "old"},
+ }}
+ localClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(oldSTS).Build()
+ r := &pullReconciler{c: NewTaskController(base.Controller{LocalKubeClient: localClient})}
+ mgmtTask := &rlarkv1alpha1.Task{ObjectMeta: metav1.ObjectMeta{
+ Name: "task", Namespace: "default", ResourceVersion: "2",
+ Annotations: map[string]string{RestartedAtAnnotation: "new"},
+ }}
+
+ required, err := r.restartCleanupRequired(context.Background(), mgmtTask, "rlark-system")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !required {
+ t.Fatal("new restart annotation should require cleanup")
+ }
+ built := buildStatefulSet(mgmtTask, &rlarkv1alpha1.KubernetesWorkloadSpec{})
+ if got := built.Annotations[RestartedAtAnnotation]; got != "new" {
+ t.Fatalf("StatefulSet restart annotation = %q, want new", got)
+ }
+
+ oldSTS.Annotations[RestartedAtAnnotation] = "new"
+ if err := localClient.Update(context.Background(), oldSTS); err != nil {
+ t.Fatal(err)
+ }
+ required, err = r.restartCleanupRequired(context.Background(), mgmtTask, "rlark-system")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if required {
+ t.Fatal("matching restart annotation should make rebuild idempotent")
+ }
+}
diff --git a/apps/rlark/pkg/agent/controllers/task/push_daemonset.go b/apps/rlark/pkg/agent/controllers/task/push_daemonset.go
index 6e97ad9..ead41e5 100644
--- a/apps/rlark/pkg/agent/controllers/task/push_daemonset.go
+++ b/apps/rlark/pkg/agent/controllers/task/push_daemonset.go
@@ -4,7 +4,9 @@ import (
"context"
"fmt"
+ "github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -60,21 +62,34 @@ func (r *pushDaemonSetReconciler) Reconcile(ctx context.Context, req reconcile.R
return reconcile.Result{}, nil
}
- phase, message := daemonSetPhase(&ds)
- pods, err := listTaskPods(ctx, r.c.LocalKubeClient, ds.Namespace, ds.Spec.Selector.MatchLabels)
- if err != nil {
- logger.Error(err, "failed to list pods")
- }
+ phase, message, pods := daemonSetPhase(ctx, logger, r.c.LocalKubeClient, &ds)
observedNodes := podNodeNames(pods)
return updateMgmtTaskStatus(ctx, logger, r.c.ManagementClient, &mgmtTask, phase, message, observedNodes)
}
-func daemonSetPhase(ds *appsv1.DaemonSet) (rlarkv1alpha1.TaskPhase, string) {
- if ds.Status.NumberReady >= ds.Status.DesiredNumberScheduled && ds.Status.DesiredNumberScheduled > 0 {
- return rlarkv1alpha1.TaskPhaseRunning, ""
+func daemonSetPhase(ctx context.Context, logger logr.Logger, localClient client.Client, ds *appsv1.DaemonSet) (rlarkv1alpha1.TaskPhase, string, []corev1.Pod) {
+ var phase rlarkv1alpha1.TaskPhase
+ var message string
+ switch {
+ case ds.Status.NumberReady >= ds.Status.DesiredNumberScheduled && ds.Status.DesiredNumberScheduled > 0:
+ phase = rlarkv1alpha1.TaskPhaseRunning
+ case ds.Status.NumberUnavailable > 0:
+ phase = rlarkv1alpha1.TaskPhaseFailed
+ message = "daemonset pods unavailable"
+ default:
+ phase = rlarkv1alpha1.TaskPhasePending
+ }
+
+ pods, err := listTaskPods(ctx, localClient, ds.Namespace, ds.Spec.Selector.MatchLabels)
+ if err != nil {
+ logger.Error(err, "failed to list pods")
}
- if ds.Status.NumberUnavailable > 0 {
- return rlarkv1alpha1.TaskPhaseFailed, "daemonset pods unavailable"
+ // Override to Failed when any pod container is in an abnormal state
+ // (CrashLoopBackOff, ImagePullBackOff, OOMKilled, etc.) so operators
+ // see the failure immediately instead of a misleading Running/Pending.
+ if podMsg, found := podFailureMessage(pods); found && phase != rlarkv1alpha1.TaskPhaseSucceeded {
+ phase = rlarkv1alpha1.TaskPhaseFailed
+ message = podMsg
}
- return rlarkv1alpha1.TaskPhasePending, ""
+ return phase, message, pods
}
diff --git a/apps/rlark/pkg/agent/controllers/task/push_deployment.go b/apps/rlark/pkg/agent/controllers/task/push_deployment.go
index c4979b9..c536af3 100644
--- a/apps/rlark/pkg/agent/controllers/task/push_deployment.go
+++ b/apps/rlark/pkg/agent/controllers/task/push_deployment.go
@@ -65,11 +65,7 @@ func (r *pushDeploymentReconciler) Reconcile(ctx context.Context, req reconcile.
return reconcile.Result{}, nil
}
- phase, message := deploymentPhase(&deploy)
- pods, err := listTaskPods(ctx, r.c.LocalKubeClient, deploy.Namespace, deploy.Spec.Selector.MatchLabels)
- if err != nil {
- logger.Error(err, "failed to list pods")
- }
+ phase, message, pods := deploymentPhase(ctx, logger, r.c.LocalKubeClient, &deploy)
observedNodes := podNodeNames(pods)
// pullProgress aggregation is now performed by the control-plane Task
// reconciler from Node.status.pullProgress, so the cluster-agent no
@@ -77,7 +73,23 @@ func (r *pushDeploymentReconciler) Reconcile(ctx context.Context, req reconcile.
return updateMgmtTaskStatus(ctx, logger, r.c.ManagementClient, &mgmtTask, phase, message, observedNodes)
}
-func deploymentPhase(deploy *appsv1.Deployment) (rlarkv1alpha1.TaskPhase, string) {
+func deploymentPhase(ctx context.Context, logger logr.Logger, localClient client.Client, deploy *appsv1.Deployment) (rlarkv1alpha1.TaskPhase, string, []corev1.Pod) {
+ phase, message := deploymentStatusPhase(deploy)
+ pods, err := listTaskPods(ctx, localClient, deploy.Namespace, deploy.Spec.Selector.MatchLabels)
+ if err != nil {
+ logger.Error(err, "failed to list pods")
+ }
+ // Override to Failed when any pod container is in an abnormal state
+ // (CrashLoopBackOff, ImagePullBackOff, OOMKilled, etc.) so operators
+ // see the failure immediately instead of a misleading Running/Pending.
+ if podMsg, found := podFailureMessage(pods); found && phase != rlarkv1alpha1.TaskPhaseSucceeded {
+ phase = rlarkv1alpha1.TaskPhaseFailed
+ message = podMsg
+ }
+ return phase, message, pods
+}
+
+func deploymentStatusPhase(deploy *appsv1.Deployment) (rlarkv1alpha1.TaskPhase, string) {
desired := computeDesiredReplicas(deploy.Spec.Replicas)
if desired == 0 {
return rlarkv1alpha1.TaskPhaseStopped, ""
@@ -96,6 +108,57 @@ func deploymentPhase(deploy *appsv1.Deployment) (rlarkv1alpha1.TaskPhase, string
return rlarkv1alpha1.TaskPhasePending, ""
}
+// abnormalContainerReasons lists Pod container waiting/terminated reasons that
+// indicate the workload will not reach Running on its own. When any container
+// of any pod backing a Task is in one of these states, the Task is marked Failed
+// so operators see the problem immediately instead of an indefinite Pending or
+// a misleading Running (a Pod can be phase=Running while a container is in
+// CrashLoopBackOff).
+var abnormalContainerReasons = map[string]struct{}{
+ // Waiting states — container cannot start or is stuck in a restart loop.
+ "ImagePullBackOff": {},
+ "ErrImagePull": {},
+ "InvalidImageName": {},
+ "CreateContainerConfigError": {},
+ "CreateContainerError": {},
+ "CrashLoopBackOff": {},
+ // Terminated states — container exited abnormally.
+ "OOMKilled": {},
+ "ContainerCannotRun": {},
+ "DeadlineExceeded": {},
+}
+
+// podFailureMessage inspects pods backing a Task for container states that
+// indicate a terminal failure (CrashLoopBackOff, ImagePullBackOff, OOMKilled,
+// etc.). If any such state is found it returns a human-readable message and
+// true; otherwise it returns "", false.
+func podFailureMessage(pods []corev1.Pod) (string, bool) {
+ for i := range pods {
+ pod := &pods[i]
+ for _, cs := range pod.Status.ContainerStatuses {
+ if waiting := cs.State.Waiting; waiting != nil {
+ if _, ok := abnormalContainerReasons[waiting.Reason]; ok {
+ detail := waiting.Message
+ if detail == "" {
+ detail = waiting.Reason
+ }
+ return fmt.Sprintf("pod %s container %s: %s", pod.Name, cs.Name, detail), true
+ }
+ }
+ if terminated := cs.State.Terminated; terminated != nil {
+ if _, ok := abnormalContainerReasons[terminated.Reason]; ok {
+ detail := terminated.Message
+ if detail == "" {
+ detail = terminated.Reason
+ }
+ return fmt.Sprintf("pod %s container %s: %s", pod.Name, cs.Name, detail), true
+ }
+ }
+ }
+ }
+ return "", false
+}
+
// --- shared helper functions for push reconcilers ---
// listTaskPods lists the local pods backing a workload via its selector labels.
diff --git a/apps/rlark/pkg/agent/controllers/task/push_statefulset.go b/apps/rlark/pkg/agent/controllers/task/push_statefulset.go
index 284b50f..ad6ae8f 100644
--- a/apps/rlark/pkg/agent/controllers/task/push_statefulset.go
+++ b/apps/rlark/pkg/agent/controllers/task/push_statefulset.go
@@ -5,11 +5,13 @@ import (
"fmt"
appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
+ "github.com/go-logr/logr"
rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1"
)
@@ -60,22 +62,34 @@ func (r *pushStatefulSetReconciler) Reconcile(ctx context.Context, req reconcile
return reconcile.Result{}, nil
}
- phase, message := statefulSetPhase(&sts)
- pods, err := listTaskPods(ctx, r.c.LocalKubeClient, sts.Namespace, sts.Spec.Selector.MatchLabels)
- if err != nil {
- logger.Error(err, "failed to list pods")
- }
+ phase, message, pods := statefulSetPhase(ctx, logger, r.c.LocalKubeClient, &sts)
observedNodes := podNodeNames(pods)
return updateMgmtTaskStatus(ctx, logger, r.c.ManagementClient, &mgmtTask, phase, message, observedNodes)
}
-func statefulSetPhase(sts *appsv1.StatefulSet) (rlarkv1alpha1.TaskPhase, string) {
+func statefulSetPhase(ctx context.Context, logger logr.Logger, localClient client.Client, sts *appsv1.StatefulSet) (rlarkv1alpha1.TaskPhase, string, []corev1.Pod) {
desired := computeDesiredReplicas(sts.Spec.Replicas)
- if desired == 0 {
- return rlarkv1alpha1.TaskPhaseStopped, ""
+ var phase rlarkv1alpha1.TaskPhase
+ switch {
+ case desired == 0:
+ phase = rlarkv1alpha1.TaskPhaseStopped
+ case sts.Status.ReadyReplicas >= desired:
+ phase = rlarkv1alpha1.TaskPhaseRunning
+ default:
+ phase = rlarkv1alpha1.TaskPhasePending
+ }
+
+ pods, err := listTaskPods(ctx, localClient, sts.Namespace, sts.Spec.Selector.MatchLabels)
+ if err != nil {
+ logger.Error(err, "failed to list pods")
}
- if sts.Status.ReadyReplicas >= desired {
- return rlarkv1alpha1.TaskPhaseRunning, ""
+ // Override to Failed when any pod container is in an abnormal state
+ // (CrashLoopBackOff, ImagePullBackOff, OOMKilled, etc.) so operators
+ // see the failure immediately instead of a misleading Running/Pending.
+ var message string
+ if podMsg, found := podFailureMessage(pods); found && phase != rlarkv1alpha1.TaskPhaseSucceeded {
+ phase = rlarkv1alpha1.TaskPhaseFailed
+ message = podMsg
}
- return rlarkv1alpha1.TaskPhasePending, ""
+ return phase, message, pods
}
diff --git a/apps/rlark/pkg/agent/node_agent.go b/apps/rlark/pkg/agent/node_agent.go
index 6c74cfa..2581b1c 100644
--- a/apps/rlark/pkg/agent/node_agent.go
+++ b/apps/rlark/pkg/agent/node_agent.go
@@ -3,8 +3,11 @@ package agent
import (
"context"
"encoding/json"
+ "net/http"
+ _ "net/http/pprof" // 注册 /debug/pprof 到 DefaultServeMux
"time"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/sync/errgroup"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -180,6 +183,23 @@ func (n *nodeAgent) Run(ctx context.Context) error {
eg.Go(func() error {
return nodeserver.Run(ctx)
})
+ // metrics/pprof HTTP server,复用 --metrics-bind-address(默认 :8081)
+ eg.Go(func() error {
+ mux := http.DefaultServeMux
+ mux.Handle("/metrics", promhttp.Handler())
+ srv := &http.Server{Addr: n.a.config.MetricsBindAddress, Handler: mux}
+ go func() {
+ <-ctx.Done()
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ _ = srv.Shutdown(shutdownCtx)
+ }()
+ logger.Info("Node agent metrics/pprof listening", "address", n.a.config.MetricsBindAddress)
+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ return err
+ }
+ return nil
+ })
if imgPuller != nil {
eg.Go(func() error {
return imgPuller.Run(ctx)
diff --git a/apps/rlark/pkg/common/registry.go b/apps/rlark/pkg/common/registry.go
new file mode 100644
index 0000000..08b636c
--- /dev/null
+++ b/apps/rlark/pkg/common/registry.go
@@ -0,0 +1,15 @@
+package common
+
+import "strings"
+
+// NormalizeRegistry makes a registry address comparable with container image
+// references. Users commonly paste a URL while Kubernetes images use a bare
+// host[:port] prefix.
+func NormalizeRegistry(value string) string {
+ value = strings.TrimSpace(value)
+ value = strings.TrimPrefix(value, "https://")
+ value = strings.TrimPrefix(value, "http://")
+ value = strings.TrimRight(value, "/")
+ value = strings.TrimSuffix(value, "/v2")
+ return strings.TrimRight(value, "/")
+}
diff --git a/apps/rlark/pkg/common/registry_test.go b/apps/rlark/pkg/common/registry_test.go
new file mode 100644
index 0000000..f2996cc
--- /dev/null
+++ b/apps/rlark/pkg/common/registry_test.go
@@ -0,0 +1,17 @@
+package common
+
+import "testing"
+
+func TestNormalizeRegistry(t *testing.T) {
+ tests := map[string]string{
+ "harbor.example.com": "harbor.example.com",
+ "https://harbor.example.com/": "harbor.example.com",
+ "http://harbor.example.com:5000": "harbor.example.com:5000",
+ "harbor.example.com/v2/": "harbor.example.com",
+ }
+ for input, want := range tests {
+ if got := NormalizeRegistry(input); got != want {
+ t.Errorf("NormalizeRegistry(%q) = %q, want %q", input, got, want)
+ }
+ }
+}
diff --git a/apps/rlark/pkg/controllermanager/job/build.go b/apps/rlark/pkg/controllermanager/job/build.go
index 39528fe..c0bba1f 100644
--- a/apps/rlark/pkg/controllermanager/job/build.go
+++ b/apps/rlark/pkg/controllermanager/job/build.go
@@ -77,6 +77,12 @@ func buildRayAnnotations(job *rlarkv1alpha1.Job, t rlarkv1alpha1.JobTaskTemplate
rlarkv1alpha1.RayTotalNodesAnnotation: strconv.Itoa(totalNodeCount(job.Spec.Tasks)),
rlarkv1alpha1.RayNodeRankStartAnnotation: strconv.Itoa(rankStartForTask(job, t.Name)),
}
+ if restartedAt := job.Annotations[RestartedAtAnnotation]; restartedAt != "" {
+ annotations[RestartedAtAnnotation] = restartedAt
+ }
+ if job.Spec.Stopped {
+ annotations[StoppedAnnotation] = "true"
+ }
if t.Head {
annotations[rlarkv1alpha1.RayRoleAnnotation] = rlarkv1alpha1.RayRoleHead
} else {
diff --git a/apps/rlark/pkg/controllermanager/job/job_controller.go b/apps/rlark/pkg/controllermanager/job/job_controller.go
index 8d1ae2e..18b4c0f 100644
--- a/apps/rlark/pkg/controllermanager/job/job_controller.go
+++ b/apps/rlark/pkg/controllermanager/job/job_controller.go
@@ -29,10 +29,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
}
// IsTerminal reports whether terminal.
-func (r *Reconciler) IsTerminal(obj client.Object) bool {
- job := obj.(*rlarkv1alpha1.Job)
- return job.Status.Phase == rlarkv1alpha1.JobPhaseSucceeded ||
- job.Status.Phase == rlarkv1alpha1.JobPhaseFailed
+func (r *Reconciler) IsTerminal(client.Object) bool {
+ return false
}
// ReconcileStateMachine reconciles the resource.
diff --git a/apps/rlark/pkg/controllermanager/job/statemachine.go b/apps/rlark/pkg/controllermanager/job/statemachine.go
index 55e45ca..2440404 100644
--- a/apps/rlark/pkg/controllermanager/job/statemachine.go
+++ b/apps/rlark/pkg/controllermanager/job/statemachine.go
@@ -11,12 +11,15 @@ import (
// Constants used by the package.
const (
+ RestartedAtAnnotation = "rlark.io/restarted-at"
+ StoppedAnnotation = "rlark.io/stopped"
+
EventInit = "init"
+ EventTasksPending = "tasks-pending"
EventTasksRunning = "tasks-running"
EventAllTasksDone = "all-tasks-succeeded"
EventAnyTaskFailed = "any-task-failed"
EventJobStopped = "job-stopped"
- EventJobResumed = "job-resumed"
)
var jobEvents = fsm.Events{
@@ -30,25 +33,35 @@ var jobEvents = fsm.Events{
Src: []string{string(rlarkv1alpha1.JobPhasePending)},
Dst: string(rlarkv1alpha1.JobPhaseRunning),
},
+ {
+ Name: EventTasksPending,
+ Src: []string{
+ string(rlarkv1alpha1.JobPhaseRunning),
+ string(rlarkv1alpha1.JobPhaseStopped),
+ string(rlarkv1alpha1.JobPhaseSucceeded),
+ string(rlarkv1alpha1.JobPhaseFailed),
+ },
+ Dst: string(rlarkv1alpha1.JobPhasePending),
+ },
{
Name: EventAllTasksDone,
- Src: []string{string(rlarkv1alpha1.JobPhaseRunning)},
+ Src: []string{string(rlarkv1alpha1.JobPhasePending), string(rlarkv1alpha1.JobPhaseRunning)},
Dst: string(rlarkv1alpha1.JobPhaseSucceeded),
},
{
Name: EventAnyTaskFailed,
- Src: []string{string(rlarkv1alpha1.JobPhaseRunning)},
+ Src: []string{string(rlarkv1alpha1.JobPhasePending), string(rlarkv1alpha1.JobPhaseRunning)},
Dst: string(rlarkv1alpha1.JobPhaseFailed),
},
{
Name: EventJobStopped,
- Src: []string{string(rlarkv1alpha1.JobPhasePending), string(rlarkv1alpha1.JobPhaseRunning), string(rlarkv1alpha1.JobPhaseFailed)},
- Dst: string(rlarkv1alpha1.JobPhaseStopped),
- },
- {
- Name: EventJobResumed,
- Src: []string{string(rlarkv1alpha1.JobPhaseStopped)},
- Dst: string(rlarkv1alpha1.JobPhasePending),
+ Src: []string{
+ string(rlarkv1alpha1.JobPhasePending),
+ string(rlarkv1alpha1.JobPhaseRunning),
+ string(rlarkv1alpha1.JobPhaseSucceeded),
+ string(rlarkv1alpha1.JobPhaseFailed),
+ },
+ Dst: string(rlarkv1alpha1.JobPhaseStopped),
},
}
@@ -86,5 +99,10 @@ func newJobStateMachine() *fsm.FSM {
now := metav1.Now()
job.Status.EndTime = &now
},
+ "enter_" + string(rlarkv1alpha1.JobPhaseStopped): func(ctx context.Context, e *fsm.Event) {
+ job := e.Args[0].(*rlarkv1alpha1.Job)
+ now := metav1.Now()
+ job.Status.EndTime = &now
+ },
})
}
diff --git a/apps/rlark/pkg/controllermanager/job/statemachine_test.go b/apps/rlark/pkg/controllermanager/job/statemachine_test.go
new file mode 100644
index 0000000..abd9ec4
--- /dev/null
+++ b/apps/rlark/pkg/controllermanager/job/statemachine_test.go
@@ -0,0 +1,77 @@
+package job
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1"
+)
+
+func TestStoppedJobRecordsEndTime(t *testing.T) {
+ job := &rlarkv1alpha1.Job{}
+ job.Status.Phase = rlarkv1alpha1.JobPhaseRunning
+ fsm := newJobStateMachine()
+ fsm.SetState(string(rlarkv1alpha1.JobPhaseRunning))
+
+ if err := fsm.Event(context.Background(), EventJobStopped, job); err != nil {
+ t.Fatalf("stop job: %v", err)
+ }
+ if job.Status.EndTime == nil {
+ t.Fatal("stopped job did not record endTime")
+ }
+}
+
+func newJob(phase rlarkv1alpha1.JobPhase) *rlarkv1alpha1.Job {
+ return &rlarkv1alpha1.Job{Status: rlarkv1alpha1.JobStatus{Phase: phase}}
+}
+
+// transition fires event from start and reports the resulting phase.
+func transition(t *testing.T, start rlarkv1alpha1.JobPhase, event string) rlarkv1alpha1.JobPhase {
+ t.Helper()
+ f := newJobStateMachine()
+ f.SetState(string(start))
+ job := newJob(start)
+ if err := f.Event(context.Background(), event, job); err != nil {
+ t.Fatalf("unexpected error firing %s from %s: %v", event, start, err)
+ }
+ return job.Status.Phase
+}
+
+// A Job stuck in Pending must be able to reach Failed when any Task fails —
+// this is the crash-loop scenario where a Task's pod enters CrashLoopBackOff
+// before the Job ever reaches Running.
+func TestJobCanFailFromPending(t *testing.T) {
+ assert.Equal(t, rlarkv1alpha1.JobPhaseFailed, transition(t, rlarkv1alpha1.JobPhasePending, EventAnyTaskFailed))
+ // Pre-existing behavior must remain intact.
+ assert.Equal(t, rlarkv1alpha1.JobPhaseFailed, transition(t, rlarkv1alpha1.JobPhaseRunning, EventAnyTaskFailed))
+}
+
+// A Job in Pending must also be able to reach Succeeded when all Tasks finish
+// before the Job reaches Running (e.g. very short-lived Tasks).
+func TestJobCanSucceedFromPending(t *testing.T) {
+ assert.Equal(t, rlarkv1alpha1.JobPhaseSucceeded, transition(t, rlarkv1alpha1.JobPhasePending, EventAllTasksDone))
+ assert.Equal(t, rlarkv1alpha1.JobPhaseSucceeded, transition(t, rlarkv1alpha1.JobPhaseRunning, EventAllTasksDone))
+}
+
+// Pending -> Running transition must still work.
+func TestJobCanRunFromPending(t *testing.T) {
+ assert.Equal(t, rlarkv1alpha1.JobPhaseRunning, transition(t, rlarkv1alpha1.JobPhasePending, EventTasksRunning))
+}
+
+// evaluateJobEvent short-circuits on AnyFailed; combined with the
+// EventAnyTaskFailed source fix, a Pending job with a failed Task must be
+// able to transition to Failed (the event is now allowed from Pending).
+func TestEvaluateJobEventFailedShortCircuitsAndFiresFromPending(t *testing.T) {
+ f := newJobStateMachine()
+ f.SetState(string(rlarkv1alpha1.JobPhasePending))
+ job := newJob(rlarkv1alpha1.JobPhasePending)
+ job.Status.Tasks = []rlarkv1alpha1.JobTaskStatus{
+ {Name: "actor", Phase: rlarkv1alpha1.TaskPhaseFailed},
+ {Name: "learner", Phase: rlarkv1alpha1.TaskPhaseRunning},
+ }
+ event := (&Reconciler{}).evaluateJobEvent(job)
+ assert.Equal(t, EventAnyTaskFailed, event)
+ assert.True(t, f.Can(event), "EventAnyTaskFailed must be allowed from Pending")
+}
diff --git a/apps/rlark/pkg/controllermanager/job/sync.go b/apps/rlark/pkg/controllermanager/job/sync.go
index 51ccaaf..8bd2f87 100644
--- a/apps/rlark/pkg/controllermanager/job/sync.go
+++ b/apps/rlark/pkg/controllermanager/job/sync.go
@@ -80,7 +80,7 @@ func (r *Reconciler) reconcileTask(
var task rlarkv1alpha1.Task
err := r.Get(ctx, types.NamespacedName{Name: taskName, Namespace: taskNamespace}, &task)
if err == nil {
- if !taskSpecEqual(&task, job, t) {
+ if !taskEqual(&task, job, t) {
desired := buildTask(job, t, taskName, taskNamespace)
task.Spec = desired.Spec
task.Annotations = desired.Annotations
@@ -108,9 +108,11 @@ func (r *Reconciler) reconcileTask(
return newTask, nil
}
-func taskSpecEqual(existing *rlarkv1alpha1.Task, job *rlarkv1alpha1.Job, t rlarkv1alpha1.JobTaskTemplate) bool {
+func taskEqual(existing *rlarkv1alpha1.Task, job *rlarkv1alpha1.Job, t rlarkv1alpha1.JobTaskTemplate) bool {
desired := buildTask(job, t, existing.Name, existing.Namespace)
- return reflect.DeepEqual(existing.Spec, desired.Spec)
+ return reflect.DeepEqual(existing.Spec, desired.Spec) &&
+ existing.Annotations[RestartedAtAnnotation] == desired.Annotations[RestartedAtAnnotation] &&
+ existing.Annotations[StoppedAnnotation] == desired.Annotations[StoppedAnnotation]
}
func (r *Reconciler) reconcileWithStateMachine(
@@ -158,14 +160,6 @@ func (r *Reconciler) reconcileWithStateMachine(
}
func (r *Reconciler) evaluateJobEvent(job *rlarkv1alpha1.Job) string {
- if job.Spec.Stopped {
- return EventJobStopped
- }
-
- if job.Status.Phase == rlarkv1alpha1.JobPhaseStopped {
- return EventJobResumed
- }
-
phases := make([]string, len(job.Status.Tasks))
for i, ts := range job.Status.Tasks {
phases[i] = string(ts.Phase)
@@ -177,6 +171,29 @@ func (r *Reconciler) evaluateJobEvent(job *rlarkv1alpha1.Job) string {
string(rlarkv1alpha1.TaskPhaseStopped),
)
+ if job.Spec.Stopped {
+ if s.AllStopped && s.HasItems {
+ return EventJobStopped
+ }
+ if s.HasItems && job.Status.Phase != rlarkv1alpha1.JobPhasePending {
+ return EventTasksPending
+ }
+ return ""
+ }
+
+ switch job.Status.Phase {
+ case rlarkv1alpha1.JobPhaseStopped:
+ return EventTasksPending
+ case rlarkv1alpha1.JobPhaseSucceeded:
+ if !s.AllSucceeded {
+ return EventTasksPending
+ }
+ case rlarkv1alpha1.JobPhaseFailed:
+ if !s.AnyFailed {
+ return EventTasksPending
+ }
+ }
+
if s.AnyFailed {
return EventAnyTaskFailed
}
@@ -185,9 +202,25 @@ func (r *Reconciler) evaluateJobEvent(job *rlarkv1alpha1.Job) string {
return EventAllTasksDone
}
- if s.AnyRunning {
+ if allTasksInPhase(phases, string(rlarkv1alpha1.TaskPhaseRunning)) {
return EventTasksRunning
}
+ if s.HasItems && job.Status.Phase != rlarkv1alpha1.JobPhasePending {
+ return EventTasksPending
+ }
+
return ""
}
+
+func allTasksInPhase(phases []string, phase string) bool {
+ if len(phases) == 0 {
+ return false
+ }
+ for _, current := range phases {
+ if current != phase {
+ return false
+ }
+ }
+ return true
+}
diff --git a/apps/rlark/pkg/controllermanager/job/sync_test.go b/apps/rlark/pkg/controllermanager/job/sync_test.go
new file mode 100644
index 0000000..3652f91
--- /dev/null
+++ b/apps/rlark/pkg/controllermanager/job/sync_test.go
@@ -0,0 +1,112 @@
+package job
+
+import (
+ "testing"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ rlarkv1alpha1 "github.com/rlinf/rlark/api/rlark.io/v1alpha1"
+)
+
+func TestEvaluateJobEventWaitsForAllTasks(t *testing.T) {
+ tests := []struct {
+ name string
+ job *rlarkv1alpha1.Job
+ want string
+ }{
+ {
+ name: "running waits for every task",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhasePending,
+ rlarkv1alpha1.TaskPhaseRunning, rlarkv1alpha1.TaskPhasePending),
+ },
+ {
+ name: "running when every task runs",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhasePending,
+ rlarkv1alpha1.TaskPhaseRunning, rlarkv1alpha1.TaskPhaseRunning),
+ want: EventTasksRunning,
+ },
+ {
+ name: "running returns to pending when tasks are mixed",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhaseRunning,
+ rlarkv1alpha1.TaskPhaseRunning, rlarkv1alpha1.TaskPhasePending),
+ want: EventTasksPending,
+ },
+ {
+ name: "stopping becomes pending while tasks are mixed",
+ job: jobWithTaskPhases(true, rlarkv1alpha1.JobPhaseRunning,
+ rlarkv1alpha1.TaskPhaseStopped, rlarkv1alpha1.TaskPhaseRunning),
+ want: EventTasksPending,
+ },
+ {
+ name: "stopped when every task stops",
+ job: jobWithTaskPhases(true, rlarkv1alpha1.JobPhasePending,
+ rlarkv1alpha1.TaskPhaseStopped, rlarkv1alpha1.TaskPhaseStopped),
+ want: EventJobStopped,
+ },
+ {
+ name: "starting becomes pending while tasks are mixed",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhaseStopped,
+ rlarkv1alpha1.TaskPhaseStopped, rlarkv1alpha1.TaskPhasePending),
+ want: EventTasksPending,
+ },
+ {
+ name: "pending may fail",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhasePending,
+ rlarkv1alpha1.TaskPhasePending, rlarkv1alpha1.TaskPhaseFailed),
+ want: EventAnyTaskFailed,
+ },
+ {
+ name: "pending may succeed",
+ job: jobWithTaskPhases(false, rlarkv1alpha1.JobPhasePending,
+ rlarkv1alpha1.TaskPhaseSucceeded, rlarkv1alpha1.TaskPhaseSucceeded),
+ want: EventAllTasksDone,
+ },
+ }
+
+ r := &Reconciler{}
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := r.evaluateJobEvent(tt.job); got != tt.want {
+ t.Fatalf("evaluateJobEvent() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestPendingCanReachTerminalStates(t *testing.T) {
+ for _, event := range []string{EventAnyTaskFailed, EventAllTasksDone} {
+ f := newJobStateMachine()
+ f.SetState(string(rlarkv1alpha1.JobPhasePending))
+ if !f.Can(event) {
+ t.Fatalf("Pending should allow event %q", event)
+ }
+ }
+}
+
+func TestBuildTaskCarriesRestartAnnotation(t *testing.T) {
+ job := &rlarkv1alpha1.Job{ObjectMeta: metav1.ObjectMeta{
+ Name: "job",
+ Annotations: map[string]string{RestartedAtAnnotation: "2026-08-21T00:00:00Z"},
+ }}
+ task := buildTask(job, rlarkv1alpha1.JobTaskTemplate{Name: "worker"}, "job-worker", "default")
+
+ if got := task.Annotations[RestartedAtAnnotation]; got != job.Annotations[RestartedAtAnnotation] {
+ t.Fatalf("restart annotation = %q, want %q", got, job.Annotations[RestartedAtAnnotation])
+ }
+ if !taskEqual(task, job, rlarkv1alpha1.JobTaskTemplate{Name: "worker"}) {
+ t.Fatal("task with the same restart annotation should be equal")
+ }
+
+ job.Annotations[RestartedAtAnnotation] = "2026-08-21T00:01:00Z"
+ if taskEqual(task, job, rlarkv1alpha1.JobTaskTemplate{Name: "worker"}) {
+ t.Fatal("task with an old restart annotation should require an update")
+ }
+}
+
+func jobWithTaskPhases(stopped bool, phase rlarkv1alpha1.JobPhase, phases ...rlarkv1alpha1.TaskPhase) *rlarkv1alpha1.Job {
+ job := &rlarkv1alpha1.Job{Spec: rlarkv1alpha1.JobSpec{Stopped: stopped}, Status: rlarkv1alpha1.JobStatus{Phase: phase}}
+ for i, taskPhase := range phases {
+ job.Status.Tasks = append(job.Status.Tasks, rlarkv1alpha1.JobTaskStatus{Name: string(rune('a' + i)), Phase: taskPhase})
+ }
+ return job
+}
diff --git a/apps/rlark/pkg/gateway/imageregistry_handler.go b/apps/rlark/pkg/gateway/imageregistry_handler.go
index 95156ed..02c2280 100644
--- a/apps/rlark/pkg/gateway/imageregistry_handler.go
+++ b/apps/rlark/pkg/gateway/imageregistry_handler.go
@@ -94,6 +94,7 @@ func (g *Gateway) handleGetImageRegistry(c *gin.Context) {
}
func buildDockerConfigJSON(registry, username, password string) ([]byte, error) {
+ registry = common.NormalizeRegistry(registry)
dockerConfig := map[string]map[string]map[string]string{
"auths": {
registry: {
@@ -113,6 +114,7 @@ func (g *Gateway) handleCreateImageRegistry(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"})
return
}
+ req.Registry = common.NormalizeRegistry(req.Registry)
configJSON, err := buildDockerConfigJSON(req.Registry, req.Username, req.Password)
if err != nil {
@@ -161,6 +163,7 @@ func (g *Gateway) handleUpdateImageRegistry(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"})
return
}
+ req.Registry = common.NormalizeRegistry(req.Registry)
secret, err := g.rawClient.CoreV1().Secrets(common.SecretNamespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
diff --git a/apps/rlark/pkg/network/nodeserver/metrics.go b/apps/rlark/pkg/network/nodeserver/metrics.go
new file mode 100644
index 0000000..cd60df2
--- /dev/null
+++ b/apps/rlark/pkg/network/nodeserver/metrics.go
@@ -0,0 +1,102 @@
+package nodeserver
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+
+ rlarkmetrics "github.com/rlinf/rlark/apps/rlark/pkg/metrics"
+)
+
+const subsystem = "nodeserver"
+
+var metrics = newNodeServerMetrics()
+
+type nodeServerMetrics struct {
+ connectionsTotal prometheus.Counter
+ connectionsActive prometheus.Gauge
+ dialTotal *prometheus.CounterVec
+ sshReconnectTotal *prometheus.CounterVec
+}
+
+func newNodeServerMetrics() *nodeServerMetrics {
+ connectionsTotal := prometheus.NewCounter(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "connections_total",
+ Help: "Total number of connections accepted on the nodeserver unix socket.",
+ },
+ )
+ connectionsActive := prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "connections_active",
+ Help: "Current number of active connections on the nodeserver unix socket.",
+ },
+ )
+ dialTotal := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "dial_total",
+ Help: "Total number of upstream dials, by type (direct/ssh) and status (success/error).",
+ },
+ []string{"type", "status"},
+ )
+ sshReconnectTotal := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "ssh_reconnect_total",
+ Help: "Total number of SSH tunnel reconnections, by domain.",
+ },
+ []string{"domain"},
+ )
+
+ prometheus.MustRegister(connectionsTotal, connectionsActive, dialTotal, sshReconnectTotal)
+ return &nodeServerMetrics{
+ connectionsTotal: connectionsTotal,
+ connectionsActive: connectionsActive,
+ dialTotal: dialTotal,
+ sshReconnectTotal: sshReconnectTotal,
+ }
+}
+
+// Metrics returns the package-level metrics singleton, for use by other
+// packages (e.g. agent/container) that need to record nodeserver-related events.
+func Metrics() *nodeServerMetrics {
+ return metrics
+}
+
+// IncConnections records a newly accepted connection.
+func (m *nodeServerMetrics) IncConnections() {
+ m.connectionsTotal.Inc()
+}
+
+// IncActive increments the active connection gauge.
+func (m *nodeServerMetrics) IncActive() {
+ m.connectionsActive.Inc()
+}
+
+// DecActive decrements the active connection gauge.
+func (m *nodeServerMetrics) DecActive() {
+ m.connectionsActive.Dec()
+}
+
+// IncDial records an upstream dial attempt.
+// dialType must be "direct" or "ssh"; status must be "success" or "error".
+func (m *nodeServerMetrics) IncDial(dialType, status string) {
+ m.dialTotal.WithLabelValues(dialType, status).Inc()
+}
+
+// IncSSHReconnect records an SSH tunnel reconnection for the given domain.
+func (m *nodeServerMetrics) IncSSHReconnect(domain string) {
+ m.sshReconnectTotal.WithLabelValues(domain).Inc()
+}
+
+// OnReconnect returns a callback suitable for SSHDialerConfig.OnReconnect.
+func OnReconnect() func(domainID string) {
+ return func(domainID string) {
+ metrics.IncSSHReconnect(domainID)
+ }
+}
diff --git a/apps/rlark/pkg/network/nodeserver/server.go b/apps/rlark/pkg/network/nodeserver/server.go
index 641c486..31a5ca1 100644
--- a/apps/rlark/pkg/network/nodeserver/server.go
+++ b/apps/rlark/pkg/network/nodeserver/server.go
@@ -106,6 +106,7 @@ func (s *NodeServer[C]) Run(ctx context.Context) error {
_ = conn.Close()
continue
}
+ metrics.IncConnections()
go s.handleConnection(ctx, utils.NewWrapConn(conn), cred)
}
}
@@ -117,6 +118,9 @@ func (s *NodeServer[C]) handleConnection(ctx context.Context, conn *utils.WrapCo
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
+ metrics.IncActive()
+ defer metrics.DecActive()
+
defer func() { _ = conn.Close() }()
network, host, port, query, err := utils.ReadTargetFromConn(conn)
@@ -169,10 +173,28 @@ func (s *NodeServer[C]) handleConnection(ctx context.Context, conn *utils.WrapCo
}
}
+ // 记录结束方向和原因,用于定位"谁在断连接"。
+ type copyResult struct {
+ direction string
+ err error
+ }
+ resultCh := make(chan copyResult, 2)
go func() {
- _, _ = io.Copy(conn2, conn)
+ _, err := io.Copy(conn2, conn) // sidecar → 上游
+ resultCh <- copyResult{direction: "sidecar->upstream", err: err}
}()
- _, _ = io.Copy(conn, conn2)
+ go func() {
+ _, err := io.Copy(conn, conn2) // 上游 → sidecar
+ resultCh <- copyResult{direction: "upstream->sidecar", err: err}
+ }()
+
+ first := <-resultCh
+ logger.Info("Forwarding connection closing",
+ "host", host, "port", port,
+ "closedBy", first.direction,
+ "err", first.err,
+ "errType", fmt.Sprintf("%T", first.err),
+ )
}
func (s *NodeServer[C]) handleGetIP(ctx *gin.Context) {
diff --git a/apps/rlark/pkg/network/sidecar/config.go b/apps/rlark/pkg/network/sidecar/config.go
index ce691a8..84531bb 100644
--- a/apps/rlark/pkg/network/sidecar/config.go
+++ b/apps/rlark/pkg/network/sidecar/config.go
@@ -31,18 +31,23 @@ type Config struct {
// HostsFile 是需要更新的 hosts 文件路径。
HostsFile string
+
+ // MetricsListenAddress 是 metrics/pprof HTTP server 的监听地址。
+ // 空值表示不启用。
+ MetricsListenAddress string
}
// DefaultConfig returns the default config.
func DefaultConfig() Config {
return Config{
- UnixSocketAddress: "/var/run/rlark/nodeserver.sock",
- TunName: "gnet0",
- TunMTU: 1500,
- ProxyListenAddress: ":5700",
- HostsSyncEnabled: true,
- HostsSyncInterval: 30 * time.Second,
- HostsFile: "/etc/hosts",
+ UnixSocketAddress: "/var/run/rlark/nodeserver.sock",
+ TunName: "gnet0",
+ TunMTU: 1500,
+ ProxyListenAddress: ":5700",
+ HostsSyncEnabled: true,
+ HostsSyncInterval: 30 * time.Second,
+ HostsFile: "/etc/hosts",
+ MetricsListenAddress: ":5790",
}
}
@@ -55,4 +60,5 @@ func (c *Config) SetupFlags(fs *pflag.FlagSet) {
fs.BoolVar(&c.HostsSyncEnabled, "sidecar-hosts-sync-enabled", c.HostsSyncEnabled, "Enable periodic hosts file sync from NodeServer")
fs.DurationVar(&c.HostsSyncInterval, "sidecar-hosts-sync-interval", c.HostsSyncInterval, "Interval between hosts sync attempts")
fs.StringVar(&c.HostsFile, "sidecar-hosts-file", c.HostsFile, "Path to the hosts file to update")
+ fs.StringVar(&c.MetricsListenAddress, "sidecar-metrics-listen", c.MetricsListenAddress, "Metrics/pprof HTTP listen address (empty=disabled)")
}
diff --git a/apps/rlark/pkg/network/sidecar/hosts.go b/apps/rlark/pkg/network/sidecar/hosts.go
index feb5055..b65c618 100644
--- a/apps/rlark/pkg/network/sidecar/hosts.go
+++ b/apps/rlark/pkg/network/sidecar/hosts.go
@@ -69,13 +69,16 @@ func (h *hostsSyncer) syncOnce(ctx context.Context) error {
hosts, err := h.fetchHosts(ctx)
if err != nil {
+ metrics.IncHostsSync("error")
return fmt.Errorf("fetch hosts: %w", err)
}
if err := h.updateHostsFile(hosts); err != nil {
+ metrics.IncHostsSync("error")
return fmt.Errorf("update hosts file: %w", err)
}
+ metrics.IncHostsSync("success")
logger.V(1).Info("Hosts file synced", "entries", len(hosts))
return nil
}
diff --git a/apps/rlark/pkg/network/sidecar/metrics.go b/apps/rlark/pkg/network/sidecar/metrics.go
new file mode 100644
index 0000000..ca38ab5
--- /dev/null
+++ b/apps/rlark/pkg/network/sidecar/metrics.go
@@ -0,0 +1,101 @@
+package sidecar
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+
+ rlarkmetrics "github.com/rlinf/rlark/apps/rlark/pkg/metrics"
+ "github.com/rlinf/rlark/apps/rlark/pkg/network/tun"
+)
+
+const subsystem = "sidecar"
+
+var metrics = newSidecarMetrics()
+
+type sidecarMetrics struct {
+ tunPacketsTotal *prometheus.CounterVec
+ proxyConnectionsTotal *prometheus.CounterVec
+ proxyConnectionsActive prometheus.Gauge
+ hostsSyncTotal *prometheus.CounterVec
+}
+
+func newSidecarMetrics() *sidecarMetrics {
+ tunPacketsTotal := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "tun_packets_total",
+ Help: "Total number of packets forwarded through the TUN device.",
+ },
+ []string{"direction"}, // tx (pod→remote) / rx (remote→pod)
+ )
+ proxyConnectionsTotal := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "proxy_connections_total",
+ Help: "Total number of inbound connections accepted by the proxy listener.",
+ },
+ []string{"protocol"}, // tcp / udp / icmp
+ )
+ proxyConnectionsActive := prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "proxy_connections_active",
+ Help: "Current number of active inbound proxy connections.",
+ },
+ )
+ hostsSyncTotal := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: rlarkmetrics.Namespace,
+ Subsystem: subsystem,
+ Name: "hosts_sync_total",
+ Help: "Total number of hosts file sync attempts.",
+ },
+ []string{"status"}, // success / error
+ )
+
+ prometheus.MustRegister(tunPacketsTotal, proxyConnectionsTotal, proxyConnectionsActive, hostsSyncTotal)
+ m := &sidecarMetrics{
+ tunPacketsTotal: tunPacketsTotal,
+ proxyConnectionsTotal: proxyConnectionsTotal,
+ proxyConnectionsActive: proxyConnectionsActive,
+ hostsSyncTotal: hostsSyncTotal,
+ }
+ // 注入到 tun 包,避免 import cycle(sidecar → tun → sidecar)
+ tun.RegisterMetrics(tunPacketsTotal, proxyConnectionsTotal, proxyConnectionsActive)
+ return m
+}
+
+// Metrics returns the package-level metrics singleton, for use by other
+// packages (e.g. network/tun) that need to record sidecar-related events.
+func Metrics() *sidecarMetrics {
+ return metrics
+}
+
+// IncTunPackets records a forwarded packet on the TUN device.
+// direction must be "tx" (pod → remote) or "rx" (remote → pod).
+func (m *sidecarMetrics) IncTunPackets(direction string) {
+ m.tunPacketsTotal.WithLabelValues(direction).Inc()
+}
+
+// IncProxyConnections records a new inbound proxy connection.
+// protocol must be "tcp", "udp", or "icmp".
+func (m *sidecarMetrics) IncProxyConnections(protocol string) {
+ m.proxyConnectionsTotal.WithLabelValues(protocol).Inc()
+}
+
+// IncProxyActive increments the active proxy connection gauge.
+func (m *sidecarMetrics) IncProxyActive() {
+ m.proxyConnectionsActive.Inc()
+}
+
+// DecProxyActive decrements the active proxy connection gauge.
+func (m *sidecarMetrics) DecProxyActive() {
+ m.proxyConnectionsActive.Dec()
+}
+
+// IncHostsSync records a hosts sync attempt. status must be "success" or "error".
+func (m *sidecarMetrics) IncHostsSync(status string) {
+ m.hostsSyncTotal.WithLabelValues(status).Inc()
+}
diff --git a/apps/rlark/pkg/network/sidecar/server.go b/apps/rlark/pkg/network/sidecar/server.go
index dc16b25..70565aa 100644
--- a/apps/rlark/pkg/network/sidecar/server.go
+++ b/apps/rlark/pkg/network/sidecar/server.go
@@ -7,8 +7,11 @@ import (
"math"
"net"
"net/http"
+ _ "net/http/pprof" // 注册 /debug/pprof 到 DefaultServeMux
"time"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+
"github.com/rlinf/rlark/apps/rlark/pkg/log"
"github.com/rlinf/rlark/apps/rlark/pkg/network/nodeserver"
"github.com/rlinf/rlark/apps/rlark/pkg/network/tun"
@@ -141,6 +144,24 @@ func (s *Sidecar) Run(ctx context.Context) error {
}
}()
+ // ─── 3.5 启动 metrics/pprof HTTP server ───
+ if s.config.MetricsListenAddress != "" {
+ mux := http.DefaultServeMux
+ mux.Handle("/metrics", promhttp.Handler())
+ metricsServer := &http.Server{Addr: s.config.MetricsListenAddress, Handler: mux}
+ go func() {
+ logger.Info("Metrics/pprof listening", "address", s.config.MetricsListenAddress)
+ if err := metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ logger.Error(nil, "Metrics server stopped", "err", err)
+ }
+ }()
+ defer func() {
+ shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer shutdownCancel()
+ _ = metricsServer.Shutdown(shutdownCtx)
+ }()
+ }
+
// ─── 4. 启动 Hosts 同步(定期从 NodeServer 获取 hosts 并更新本地 hosts 文件) ───
if s.config.HostsSyncEnabled {
hs := newHostsSyncer(s.transport, s.config.HostsFile, s.config.HostsSyncInterval)
diff --git a/apps/rlark/pkg/network/tun/metrics.go b/apps/rlark/pkg/network/tun/metrics.go
new file mode 100644
index 0000000..10736bf
--- /dev/null
+++ b/apps/rlark/pkg/network/tun/metrics.go
@@ -0,0 +1,58 @@
+package tun
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// tunMetrics holds the metrics for the tun package. It is populated via
+// RegisterMetrics to avoid an import cycle with the sidecar package
+// (sidecar → tun → sidecar).
+var tunMetrics = &tunMetricsType{}
+
+type tunMetricsType struct {
+ packetsTotal *prometheus.CounterVec
+ proxyConnectionsTotal *prometheus.CounterVec
+ proxyConnectionsActive prometheus.Gauge
+}
+
+// RegisterMetrics injects the sidecar-owned metric vectors so the tun package
+// can record events without importing the sidecar package.
+func RegisterMetrics(
+ packetsTotal *prometheus.CounterVec,
+ proxyConnectionsTotal *prometheus.CounterVec,
+ proxyConnectionsActive prometheus.Gauge,
+) {
+ tunMetrics.packetsTotal = packetsTotal
+ tunMetrics.proxyConnectionsTotal = proxyConnectionsTotal
+ tunMetrics.proxyConnectionsActive = proxyConnectionsActive
+}
+
+// IncPackets records a forwarded packet on the TUN device.
+// direction must be "tx" (pod → remote) or "rx" (remote → pod).
+func (m *tunMetricsType) IncPackets(direction string) {
+ if m.packetsTotal != nil {
+ m.packetsTotal.WithLabelValues(direction).Inc()
+ }
+}
+
+// IncProxyConnections records a new inbound proxy connection.
+// protocol must be "tcp", "udp", or "icmp".
+func (m *tunMetricsType) IncProxyConnections(protocol string) {
+ if m.proxyConnectionsTotal != nil {
+ m.proxyConnectionsTotal.WithLabelValues(protocol).Inc()
+ }
+}
+
+// IncProxyActive increments the active proxy connection gauge.
+func (m *tunMetricsType) IncProxyActive() {
+ if m.proxyConnectionsActive != nil {
+ m.proxyConnectionsActive.Inc()
+ }
+}
+
+// DecProxyActive decrements the active proxy connection gauge.
+func (m *tunMetricsType) DecProxyActive() {
+ if m.proxyConnectionsActive != nil {
+ m.proxyConnectionsActive.Dec()
+ }
+}
diff --git a/apps/rlark/pkg/network/tun/netstack.go b/apps/rlark/pkg/network/tun/netstack.go
index 0455c6d..a56de99 100644
--- a/apps/rlark/pkg/network/tun/netstack.go
+++ b/apps/rlark/pkg/network/tun/netstack.go
@@ -180,6 +180,7 @@ func (ns *netstack) handleRecv(tunnelConn net.Conn, ep *channel.Endpoint, wg *sy
Payload: buffer.MakeWithData(data),
})
ep.InjectInbound(ipv4.ProtocolNumber, pkt)
+ tunMetrics.IncPackets("rx")
default:
// ignore unsupported versions
}
@@ -212,6 +213,7 @@ func (ns *netstack) handleSend(ep *channel.Endpoint, tunnelConn net.Conn, wg *sy
logger.Error(nil, "Failed to send packet to tunnel", "err", err)
return
}
+ tunMetrics.IncPackets("tx")
}
}
diff --git a/apps/rlark/pkg/network/tun/proxy.go b/apps/rlark/pkg/network/tun/proxy.go
index 8ba4df5..8e34f87 100644
--- a/apps/rlark/pkg/network/tun/proxy.go
+++ b/apps/rlark/pkg/network/tun/proxy.go
@@ -58,12 +58,21 @@ func (p *Proxy) handleConnection(conn *utils.WrapConn) {
switch network {
case "tcp", "tcp4", "tcp6":
logger.V(1).Info("Handling TCP proxy connection", "host", host, "port", port)
+ tunMetrics.IncProxyConnections("tcp")
+ tunMetrics.IncProxyActive()
+ defer tunMetrics.DecProxyActive()
p.handleTCP(conn, host, port)
case "udp", "udp4", "udp6":
logger.V(1).Info("Handling UDP proxy connection", "host", host, "port", port)
+ tunMetrics.IncProxyConnections("udp")
+ tunMetrics.IncProxyActive()
+ defer tunMetrics.DecProxyActive()
p.handleUDP(conn, host, port)
case "icmp", "icmp4":
logger.V(1).Info("Handling ICMP proxy connection", "host", host)
+ tunMetrics.IncProxyConnections("icmp")
+ tunMetrics.IncProxyActive()
+ defer tunMetrics.DecProxyActive()
p.handleICMP(conn, host)
default:
logger.Error(nil, "Handling proxy: unsupported network protocol", "network", network)
diff --git a/apps/rlark/pkg/server/ssh_server.go b/apps/rlark/pkg/server/ssh_server.go
index 0440e3f..0e108a1 100644
--- a/apps/rlark/pkg/server/ssh_server.go
+++ b/apps/rlark/pkg/server/ssh_server.go
@@ -135,9 +135,15 @@ func (s *Server) sshPublicKeyAuth() ssh.PublicKeyHandler {
if keyID == "" {
return nil, fmt.Errorf("public key not found for user %s", conn.User())
}
+
+ realUser := conn.User()
+ if parts := strings.SplitN(keyID, "-", 2); len(parts) > 0 && parts[0] != "" {
+ realUser = parts[0]
+ }
+
_, meta, _ := s.parseSignRequest(&SignRequest{
Role: "ssh-guest",
- ClientID: conn.User(),
+ ClientID: realUser,
KeyID: keyID,
})
sshCert := &gossh.Certificate{}
@@ -261,20 +267,17 @@ func (s *Server) authenticateUserKeyFromSecret(username string, key gossh.Public
return "", err
}
- raw, ok := secret.Data[username]
- if !ok {
- return "", nil
- }
-
- for i, line := range strings.Split(string(raw), "\n") {
- line = strings.TrimSpace(line)
- if line == "" {
- continue
- }
+ for name, raw := range secret.Data {
+ for i, line := range strings.Split(string(raw), "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
- pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(line))
- if err == nil && ssh.KeysEqual(key, pubKey) {
- return fmt.Sprintf("%s-%d", username, i), nil
+ pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(line))
+ if err == nil && ssh.KeysEqual(key, pubKey) {
+ return fmt.Sprintf("%s-%d", name, i), nil
+ }
}
}