From 00194fbc2f96fc83e49702b3d958bc8aa33ef400 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:19:29 -0500 Subject: [PATCH 1/6] Sources: each step's in and out counts from the dry run (#104) The dry run (/sources/validate) answers with steps: {id: {in, out}}, what each step took and passed on over all the stored samples: frames and records for the input, records or observations for the rest, and what an output delivered. Co-Authored-By: Claude Opus 5.5 --- crates/ot-server/src/api.rs | 2 ++ crates/ot-source/src/pipeline.rs | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index 6f17998..56d33a0 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -561,11 +561,13 @@ async fn validate_source( None => pipeline.flush(last, true, ®istry), }; observations.extend(flushed.observations); + let steps = pipeline.step_counts(); let counts: BTreeMap = pipeline.take_counts().pairs().into_iter().collect(); let mut body = json!({ "valid": valid, "problems": problems, "counts": counts, + "steps": steps, "errors": errors, "observations": observations, }); diff --git a/crates/ot-source/src/pipeline.rs b/crates/ot-source/src/pipeline.rs index a1e9a0b..cb6ba62 100644 --- a/crates/ot-source/src/pipeline.rs +++ b/crates/ot-source/src/pipeline.rs @@ -305,6 +305,8 @@ pub struct Pipeline { /// Extension field types for the mappings' schema version. schema: Option, pub counts: Counts, + /// What each step took and passed on, in `spec.nodes` order. + step_counts: Vec, /// Run before the Output when no Entity links step comes before it. default_registry: RegistryStage, /// The last value sent to each (entity, field), so a feed repeating a @@ -338,6 +340,16 @@ enum Run { Nats(String), } +/// What one step took in and passed on: frames and records for the input, +/// records or observations for the rest, and what an output delivered. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] +pub struct StepCount { + #[serde(rename = "in")] + pub taken: u64, + #[serde(rename = "out")] + pub passed: u64, +} + /// What passes between steps: a record, or an observation's JSON form. #[derive(Clone)] struct Item { @@ -464,6 +476,7 @@ impl Pipeline { .copied() .filter(|&i| matches!(steps[i], Run::Map { .. })) .or_else(|| steps.iter().position(|s| matches!(s, Run::Map { .. }))); + let spec_len = spec.nodes.len(); Ok(Self { source_id: source_id.into(), codec: Codec::new(spec.codec.clone()) @@ -477,6 +490,7 @@ impl Pipeline { order, schema: None, counts: Counts::default(), + step_counts: vec![StepCount::default(); spec_len], default_registry: RegistryStage::default(), entity_sent: HashMap::new(), keep_plots: false, @@ -547,6 +561,17 @@ impl Pipeline { Duration::from_secs(secs) } + /// What each step took in and passed on so far, by step id (for a dry + /// run: the counts on the editor's steps and connections). + pub fn step_counts(&self) -> std::collections::BTreeMap { + self.spec + .nodes + .iter() + .zip(&self.step_counts) + .map(|(n, c)| (n.id.clone(), *c)) + .collect() + } + /// Take and reset the counters. pub fn take_counts(&mut self) -> Counts { std::mem::take(&mut self.counts) @@ -596,6 +621,7 @@ impl Pipeline { ) -> Output { let mut out = Output::default(); self.counts.frames += 1; + self.step_counts[self.input].taken += 1; if let Some(t) = trace.as_deref_mut() { t.start_frame(); } @@ -675,6 +701,7 @@ impl Pipeline { out: &mut Output, trace: &mut Option<&mut Trace>, ) { + self.step_counts[at].passed += 1; let n = self.children[at].len(); for i in 0..n { let c = self.children[at][i]; @@ -696,6 +723,7 @@ impl Pipeline { out: &mut Output, trace: &mut Option<&mut Trace>, ) { + self.step_counts[at].taken += 1; let id = self.spec.nodes[at].id.clone(); let mut sample = match (trace.as_deref_mut(), item.sample) { (Some(t), Some(s)) => t.samples.get_mut(s), @@ -821,6 +849,7 @@ impl Pipeline { return; }; self.counts.emitted += 1; + self.step_counts[at].passed += 1; if let Some(f) = sample { f.obs(&id, &obs); f.emitted.push(obs.clone()); @@ -830,6 +859,7 @@ impl Pipeline { } Run::Nats(subject) => { self.counts.published += 1; + self.step_counts[at].passed += 1; if let Some(f) = sample { f.item(&id, || item.value.clone()); } @@ -1342,6 +1372,34 @@ mod tests { ); } + #[test] + fn each_step_counts_what_it_took_and_passed_on() { + let spec = flow( + json!([ + {"id": "m", "type": "map", "config": pos_map()}, + {"id": "f", "type": "filter", "config": {"drop_if": {"path": "position.latitude", "gt": 33}}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "raw"}} + ]), + json!([ + {"source": "input", "target": "m"}, {"source": "m", "target": "f"}, + {"source": "f", "target": "out"}, {"source": "m", "target": "raw"} + ]), + ); + let mut p = Pipeline::new("ais", spec).unwrap(); + let body = json!([{"id": 1, "lat": 32.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}, + {"id": 2, "lat": 40.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}, + {"nothing": true}]); + p.process(&Frame::new(body.to_string().into_bytes()), &Reg::new()); + let c = p.step_counts(); + let at = |id: &str| (c[id].taken, c[id].passed); + assert_eq!(at("input"), (1, 3), "one frame, three records"); + assert_eq!(at("m"), (3, 2), "one record matched no rule"); + assert_eq!(at("f"), (2, 1)); + assert_eq!(at("out"), (1, 1)); + assert_eq!(at("raw"), (2, 2)); + } + #[test] fn a_trace_follows_each_branch() { let spec = flow( From 8cff0a3288c2bf3b5e03a95bd2a6ae43e4f83080 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:21:59 -0500 Subject: [PATCH 2/6] Sources: samples in a tray under the workspace, counts on steps and connections (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The selected step's samples are in a Tray along the bottom of the workspace: a RecordTable, one row per sample, with what the step added or changed tinted and what it removed struck through, beside what reached it; or each sample as JSON. The settings panel holds settings only. - Each step shows in → out from the dry run, and each connection what crossed it; a step that takes samples and passes none on is amber. Needs stareSDK 0.2.32 (Tray, RecordTable). Co-Authored-By: Claude Opus 5.5 --- ui/src/api/client.ts | 6 +- ui/src/index.css | 31 ++++++- ui/src/lib/trace.test.ts | 39 ++++++++- ui/src/lib/trace.ts | 37 ++++++++ ui/src/pages/sources/SourceEditor.tsx | 21 ++++- ui/src/pages/sources/flow/NodePanel.tsx | 38 +++----- ui/src/pages/sources/flow/SamplesTray.tsx | 100 ++++++++++++++++++++++ ui/src/pages/sources/flow/StepNode.tsx | 11 ++- 8 files changed, 248 insertions(+), 35 deletions(-) create mode 100644 ui/src/pages/sources/flow/SamplesTray.tsx diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 18e1f64..d31a2e9 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -999,8 +999,10 @@ export interface PreviewObservation extends Observation { export interface PreviewResult { /** Whether the source is complete; a draft is dry-run as far as it can be. */ valid: boolean - /** What stops the source running, each on its stage. */ + /** What stops the source running, each on its step. */ problems?: SourceProblem[] + /** What each step took in and passed on over all the stored samples, by step id. */ + steps?: Record counts?: Record errors?: string[] observations?: PreviewObservation[] @@ -1029,7 +1031,7 @@ export interface TraceFrameView { export interface TraceSample { /** Index in `frames` of the frame it came from. */ frame: number - /** The pipeline's stages after the transport, in order (ids as in `pipelineStages`), ending with `publish`. */ + /** The flow's steps, input first, in the order they are met (by step id), ending with `publish`. */ stages: TraceStage[] } diff --git a/ui/src/index.css b/ui/src/index.css index e0e4ddd..b4dffb2 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -935,10 +935,39 @@ body { .source-workspace { flex: 1; min-height: 0; + display: flex; + flex-direction: column; + border: 1px solid var(--color-glass-border); + border-radius: var(--radius-md); + overflow: hidden; } .source-workspace-canvas { - height: 100%; + flex: 1; + min-height: 0; +} + +.samples-tray-body { + display: flex; + flex-direction: column; + gap: var(--space-xs); + padding: var(--space-xs) 0; +} + +.samples-tray-body > .value-row, +.samples-tray-body > .error-text { + padding: 0 var(--space-md); +} + +/* What the dry run passed through a step: in → out. */ +.flow-step-counts { + font-size: 10px; + font-family: var(--font-mono); + color: var(--color-text-secondary); +} + +.flow-step-counts[data-stops] { + color: var(--status-warning); } .source-editor-json { diff --git a/ui/src/lib/trace.test.ts b/ui/src/lib/trace.test.ts index 7442845..8bdabaa 100644 --- a/ui/src/lib/trace.test.ts +++ b/ui/src/lib/trace.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { PreviewTrace } from '../api/client' -import { countsLine, samplesAt } from './trace' +import { countsLine, sampleRows, samplesAt } from './trace' // Frame 0 decodes to three records (samples #1–#3), frame 1 does not decode (#4), frame 2 is // binary (#5), frame 3 gives #6. @@ -133,3 +133,40 @@ describe('big frames and stage notes', () => { expect(atPublish.blocks).toHaveLength(1) }) }) + +describe('sampleRows', () => { + const trace = { + frames: [{ format: 'json' as const, bytes: 10, content: {} }], + samples: [ + { + frame: 0, + stages: [ + { id: 'input', items: [{ n: 1 }] }, + { id: 'map', items: [{ key: '1', lat: 1 }] }, + { id: 'filter', items: [{ key: '1', lat: 1 }] }, + ], + }, + { + frame: 0, + stages: [ + { id: 'input', items: [{ n: 2 }] }, + { id: 'map', items: [{ key: '2', lat: 40 }] }, + { id: 'filter', items: [], dropped: ['drop_if matched'] }, + ], + }, + { frame: 0, stages: [{ id: 'input', items: [], dropped: ['decode error: bad'] }, { id: 'map', items: [] }, { id: 'filter', items: [] }] }, + ], + } + + it('puts each sample beside what reached the step, and says why one did not come out', () => { + const rows = sampleRows(trace, 'filter', 'map', ['input', 'map', 'filter']) + expect(rows[0]).toEqual({ id: '0-0', label: '#1', value: { key: '1', lat: 1 }, before: { key: '1', lat: 1 } }) + expect(rows[1]).toEqual({ id: '1', label: '#2', dropped: 'dropped by Filter: drop_if matched' }) + expect(rows[2].dropped).toBe('dropped by Input: decode error: bad') + }) + + it("shows the input's records with no step before, and its decode errors", () => { + const rows = sampleRows(trace, 'input', undefined, ['input']) + expect(rows.map((r) => r.value ?? r.dropped)).toEqual([{ n: 1 }, { n: 2 }, 'decode error: bad']) + }) +}) diff --git a/ui/src/lib/trace.ts b/ui/src/lib/trace.ts index 32d8dee..fa1b8e8 100644 --- a/ui/src/lib/trace.ts +++ b/ui/src/lib/trace.ts @@ -102,6 +102,43 @@ export function samplesAt(trace: PreviewTrace | undefined, stageId: string, path }) } +/** One traced sample at a step, for a table: what the step passed on, and what reached it. */ +export interface SampleRow { + id: string + label: string + /** What the step passed on (one row per item when it passed on several). */ + value?: unknown + /** What reached the step, for what it changed. */ + before?: unknown + /** Why nothing came out here: dropped where and why, held, or not in the flow. */ + dropped?: string +} + +/** + * The traced samples at `stageId` as table rows, each beside what its step before (`parentId`) + * passed on. A sample with nothing at the step says why, from the steps on `path`. + */ +export function sampleRows(trace: PreviewTrace | undefined, stageId: string, parentId?: string, path?: string[]): SampleRow[] { + // The input's samples are frames in `samplesAt`; here they are the records it decoded. + const views = stageId === 'input' ? [] : samplesAt(trace, stageId, path) + const samples = (trace?.samples ?? []).slice(0, TRACED_SAMPLES) + return samples.flatMap((s, i): SampleRow[] => { + const at = s.stages.find((x) => x.id === stageId) + const before = parentId ? s.stages.find((x) => x.id === parentId)?.items : undefined + const items = at?.items ?? [] + if (!items.length) { + const why = (stageId === 'input' ? (at?.dropped ?? []) : (views[i]?.notes ?? [])).join(' · ') || 'Nothing came out of this step.' + return [{ id: `${i}`, label: `#${i + 1}`, dropped: at ? why : 'This step was not in the flow the preview ran.' }] + } + return items.map((item, j) => ({ + id: `${i}-${j}`, + label: items.length > 1 ? `#${i + 1}.${j + 1}` : `#${i + 1}`, + value: item, + before: before ? (before[j] ?? before[0]) : undefined, + })) + }) +} + /** The counts on one line, as `frames 5 · records 5 · emitted 4`. */ export function countsLine(result: PreviewResult | null): string { const counts = Object.entries(result?.counts ?? {}).filter(([k]) => !k.startsWith('rejected:') && !k.startsWith('grade:')) diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 2d89ffb..d4eb082 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -23,6 +23,7 @@ import { SOURCE_STATES_INFO, sourceState } from '../../lib/sourceState' import { MAX_TRACED_SAMPLES, TRACED_SAMPLES } from '../../lib/trace' import { FeedPanel } from './flow/FeedPanel' import { NodePanel } from './flow/NodePanel' +import { SamplesTray } from './flow/SamplesTray' import { StepNode, type StepView } from './flow/StepNode' import { stepIcon } from './flow/stepIcons' import { @@ -155,6 +156,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [probe, setProbe] = useState(null) const [capturing, setCapturing] = useState(false) const [nonce, setNonce] = useState(0) + const [trayOpen, setTrayOpen] = useState(true) // A source from before flows has no places for its steps: they are laid out once, as loaded. const adopt = useCallback((r: SourceRow) => { @@ -219,17 +221,20 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o title: nodeTitle(spec, n), summary: nodeSummary(spec, n), problems: byNode.byNode[n.id] ?? [], + counts: live.preview?.steps?.[n.id], } satisfies StepView, })) - }, [spec, flow, current, byNode]) + }, [spec, flow, current, byNode, live.preview]) const edges: FlowCanvasEdge[] = useMemo( () => flow.edges.map((e) => { const eid = `${e.source}->${e.target}` - return { id: eid, source: e.source, target: e.target, selected: selectedEdges.has(eid) } + // What crossed it in the dry run: what its target took in. + const crossed = live.preview?.steps?.[e.target]?.in + return { id: eid, source: e.source, target: e.target, selected: selectedEdges.has(eid), label: crossed === undefined ? undefined : String(crossed) } }), - [flow, selectedEdges], + [flow, selectedEdges, live.preview], ) const transportType = spec?.transport.type @@ -460,6 +465,15 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o height="100%" /> + {/* A step's settings (or the feed's) open over the right of the page, resizable. */} = { } /** - * The settings of one step, and its samples as they leave it. The input's are the source's - * transport and codec; the others' are the step's own. + * The settings of one step. The input's are the source's transport and codec; the others' are the + * step's own. Its samples are in the tray under the workspace. */ export function NodePanel({ spec, @@ -39,7 +38,6 @@ export function NodePanel({ schema, plugins, preview, - previewError, probe, capturing, readOnly, @@ -56,7 +54,6 @@ export function NodePanel({ schema: SchemaOverview | null plugins: PluginInfo[] preview: PreviewResult | null - previewError: string | null probe: ProbeResult | null capturing: boolean readOnly: boolean @@ -234,28 +231,17 @@ export function NodePanel({ {settings} )} -
-

- Samples - - A dry run of the source as edited (not yet saved) over its stored samples, as they leave this step. Nothing is published. + {node.type === 'input' && ( +
+ + + Connect with these settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the + source's stored samples. -

- - {node.type === 'input' && ( - <> - - - Connect with these settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the - source's stored samples. - - - )} -
- {previewError &&
{previewError}
} - {preview && } + + )} ) diff --git a/ui/src/pages/sources/flow/SamplesTray.tsx b/ui/src/pages/sources/flow/SamplesTray.tsx new file mode 100644 index 0000000..76ae764 --- /dev/null +++ b/ui/src/pages/sources/flow/SamplesTray.tsx @@ -0,0 +1,100 @@ +import { useState } from 'react' +import { RecordTable, SegmentedControl, Tray } from '@kineticlogic/staresdk' +import type { PreviewResult } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { PIPELINE_COUNTS_INFO } from '../../../lib/sourceState' +import { countsLine, sampleRows } from '../../../lib/trace' +import { StageSamples } from '../designer/StageSamples' +import type { FlowNode } from './sourceFlow' + +/** Fields shown first in a sample table, when the samples have them. */ +const PINNED = ['source_track_key', 'name', 'observed_at', 'position.latitude', 'position.longitude'] + +/** + * The selected step's samples, in a tray along the bottom of the workspace: a dry run of the source + * as edited over its stored samples, as they leave the step. As a table, each sample is a row with + * what the step changed from what reached it highlighted; as JSON, each sample whole. + */ +export function SamplesTray({ + node, + title, + path, + preview, + error, + open, + onOpenChange, +}: { + node: FlowNode | null + title: string + /** The steps from the input to the selected one. */ + path: string[] + preview: PreviewResult | null + error: string | null + open: boolean + onOpenChange: (open: boolean) => void +}) { + const [view, setView] = useState<'table' | 'json'>('table') + const [selected, setSelected] = useState(null) + const parent = path.length > 1 ? path[path.length - 2] : undefined + const rows = node && preview ? sampleRows(preview.trace, node.id, parent, path) : [] + const counts = countsLine(preview) + + return ( + + + A dry run of the source as edited (not yet saved) over its stored samples, as they leave the selected step. In the table, + fields the step added or changed are tinted and ones it removed are struck through. Nothing is published. + + + + } + > +
+ {error &&
{error}
} + {view === 'table' && counts && ( +
+ {counts} + What the source did with all the stored samples. {PIPELINE_COUNTS_INFO} +
+ )} + {view === 'table' && [...new Set(preview?.errors ?? [])].map((e) => ( +
+ {e} +
+ ))} + {!node ? null : !preview ? null : view === 'table' ? ( + + ) : ( + + )} +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/StepNode.tsx b/ui/src/pages/sources/flow/StepNode.tsx index cc09fc2..57e1006 100644 --- a/ui/src/pages/sources/flow/StepNode.tsx +++ b/ui/src/pages/sources/flow/StepNode.tsx @@ -9,6 +9,8 @@ export interface StepView extends Record { title: string summary: string problems: string[] + /** What the dry run passed through it: taken in and passed on. */ + counts?: { in: number; out: number } } /** @@ -19,12 +21,14 @@ export interface StepView extends Record { export function StepNode({ data, selected, isConnectable }: NodeProps) { const step = data as StepView const failing = step.problems.length > 0 + // A step that takes samples and passes none on is worth a look (a Filter that drops everything). + const stops = !!step.counts && step.counts.in > 0 && step.counts.out === 0 return (
{takes(step.type) && } @@ -36,6 +40,11 @@ export function StepNode({ data, selected, isConnectable }: NodeProps) { {KIND_WORD[step.type]}
{step.summary && {step.summary}} + {step.counts && ( + + {step.counts.in} → {step.counts.out} + + )} {failing && ( {step.problems[0]} From edb2649f1fbdffb2457e280700e4a5261f22ad2a Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:30:27 -0500 Subject: [PATCH 3/6] Sources: no change tints on the input's or a Map's samples (#104) A Map makes observations out of records, so every field would show as changed. Co-Authored-By: Claude Opus 5.5 --- ui/src/pages/sources/flow/SamplesTray.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/src/pages/sources/flow/SamplesTray.tsx b/ui/src/pages/sources/flow/SamplesTray.tsx index 76ae764..b40914f 100644 --- a/ui/src/pages/sources/flow/SamplesTray.tsx +++ b/ui/src/pages/sources/flow/SamplesTray.tsx @@ -86,7 +86,8 @@ export function SamplesTray({ aria-label={`Samples after ${title}`} rows={rows} pinned={PINNED} - showChanges={node.type !== 'input'} + // A Map makes observations out of records: everything would differ. + showChanges={node.type !== 'input' && node.type !== 'map'} selectedId={selected} onRowClick={setSelected} empty="No stored samples. Capture some on the input." From a9c84daf525af49d019dab1278a3bec54bd93bdc Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:31:44 -0500 Subject: [PATCH 4/6] Sources: closing a step's settings keeps it selected, its samples in the tray (#104) Co-Authored-By: Claude Opus 5.5 --- ui/src/pages/sources/SourceEditor.tsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index d4eb082..3b68946 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -145,6 +145,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [selected, setSelected] = useState(null) const [selectedEdges, setSelectedEdges] = useState>(new Set()) const [feedOpen, setFeedOpen] = useState(false) + // A step's settings can be closed and the step stay selected, its samples in the tray. + const [settingsOpen, setSettingsOpen] = useState(false) const [view, setView] = useState<'flow' | 'json'>('flow') const [jsonText, setJsonText] = useState('') const [jsonBox, jsonHeight] = useBoxHeight() @@ -256,6 +258,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const on = picks.find((c) => c.selected) if (on) { setSelected(on.id) + setSettingsOpen(true) setFeedOpen(false) } else if (picks.some((c) => c.id === selected)) setSelected(null) } @@ -316,6 +319,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o if (config !== undefined) node.config = config editFlow({ ...flow, nodes: [...flow.nodes, node] }) setSelected(node.id) + setSettingsOpen(true) setFeedOpen(false) } @@ -435,7 +439,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o icon={} onClick={() => { setFeedOpen((o) => !o) - setSelected(null) + setSettingsOpen(false) }} > Feed @@ -476,7 +480,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o /> {/* A step's settings (or the feed's) open over the right of the page, resizable. */} { if (e.key === 'Escape') { - setSelected(null) + setSettingsOpen(false) setFeedOpen(false) } }} > - {node ? ( + {node && settingsOpen ? ( setConfig(node.id, c)} onCapture={() => void capture()} - onClose={() => setSelected(null)} + onClose={() => setSettingsOpen(false)} /> ) : feedOpen ? ( setFeedOpen(false)} /> From 4912301944235e656b8fec311a7848f371516550 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:33:58 -0500 Subject: [PATCH 5/6] Sources: follow one sample through the flow (#104) Clicking a sample in the tray follows it: the steps it passed light up, the connections it took are drawn in the accent colour, the step that dropped or held it says why, and the steps it never reached fade. The tray says which sample is followed; clicking it again, or the badge's remove, stops. Co-Authored-By: Claude Opus 5.5 --- ui/src/index.css | 28 +++++++++++++++++++++ ui/src/lib/trace.test.ts | 30 ++++++++++++++++++++++- ui/src/lib/trace.ts | 16 ++++++++++++ ui/src/pages/sources/SourceEditor.tsx | 25 ++++++++++++++++--- ui/src/pages/sources/flow/SamplesTray.tsx | 21 +++++++++++++--- ui/src/pages/sources/flow/StepNode.tsx | 25 ++++++++++++++++--- 6 files changed, 133 insertions(+), 12 deletions(-) diff --git a/ui/src/index.css b/ui/src/index.css index b4dffb2..fa6b8f5 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -970,6 +970,34 @@ body { color: var(--status-warning); } +/* Following a sample: the steps it never reached fade, the step that dropped it says why. */ +.flow-step[data-trail='absent'] { + opacity: 0.4; +} + +.flow-step-trail { + font-size: 10px; + line-height: 1.3; + color: var(--color-destructive); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.flow-step-trail[data-state='held'] { + color: var(--status-warning); +} + +.react-flow__edge.flow-edge-taken .react-flow__edge-path { + stroke: var(--color-accent); + stroke-width: 2; +} + +.react-flow__edge.flow-edge-faded { + opacity: 0.3; +} + .source-editor-json { flex: 1; min-height: 0; diff --git a/ui/src/lib/trace.test.ts b/ui/src/lib/trace.test.ts index 8bdabaa..2a5086e 100644 --- a/ui/src/lib/trace.test.ts +++ b/ui/src/lib/trace.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { PreviewTrace } from '../api/client' -import { countsLine, sampleRows, samplesAt } from './trace' +import { countsLine, sampleRows, sampleTrail, samplesAt } from './trace' // Frame 0 decodes to three records (samples #1–#3), frame 1 does not decode (#4), frame 2 is // binary (#5), frame 3 gives #6. @@ -170,3 +170,31 @@ describe('sampleRows', () => { expect(rows.map((r) => r.value ?? r.dropped)).toEqual([{ n: 1 }, { n: 2 }, 'decode error: bad']) }) }) + +describe('sampleTrail', () => { + it('marks the steps a sample passed and the one that dropped it', () => { + const trace = { + frames: [], + samples: [ + { + frame: 0, + stages: [ + { id: 'input', items: [{ n: 1 }] }, + { id: 'map', items: [{ k: 1 }] }, + { id: 'f', items: [], dropped: ['drop_if matched'] }, + { id: 'raw', items: [{ k: 1 }] }, + { id: 'out', items: [] }, + { id: 'publish', items: [] }, + ], + }, + ], + } + expect(sampleTrail(trace, 0)).toEqual({ + input: { state: 'passed' }, + map: { state: 'passed' }, + f: { state: 'dropped', note: 'drop_if matched' }, + raw: { state: 'passed' }, + }) + expect(sampleTrail(trace, 5)).toEqual({}) + }) +}) diff --git a/ui/src/lib/trace.ts b/ui/src/lib/trace.ts index fa1b8e8..9024e74 100644 --- a/ui/src/lib/trace.ts +++ b/ui/src/lib/trace.ts @@ -139,6 +139,22 @@ export function sampleRows(trace: PreviewTrace | undefined, stageId: string, par }) } +/** Where one traced sample went: each step it passed, the one that dropped or held it, and why. */ +export type TrailMark = { state: 'passed' | 'dropped' | 'held'; note?: string } + +/** The trail of traced sample `index` (as `sampleRows` numbers them, from 0), by step id. */ +export function sampleTrail(trace: PreviewTrace | undefined, index: number): Record { + const s = trace?.samples[index] + const out: Record = {} + for (const x of s?.stages ?? []) { + if (x.id === 'publish') continue + if (x.dropped?.length) out[x.id] = { state: 'dropped', note: x.dropped.join(' · ') } + else if (x.held) out[x.id] = { state: 'held', note: 'waiting for the rest of its scan' } + else if (x.items.length) out[x.id] = { state: 'passed' } + } + return out +} + /** The counts on one line, as `frames 5 · records 5 · emitted 4`. */ export function countsLine(result: PreviewResult | null): string { const counts = Object.entries(result?.counts ?? {}).filter(([k]) => !k.startsWith('rejected:') && !k.startsWith('grade:')) diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 3b68946..90dc350 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -20,7 +20,7 @@ import { useCan } from '../../auth/context' import { InfoTip } from '../../components/InfoTip' import { errorMessage } from '../../lib/format' import { SOURCE_STATES_INFO, sourceState } from '../../lib/sourceState' -import { MAX_TRACED_SAMPLES, TRACED_SAMPLES } from '../../lib/trace' +import { MAX_TRACED_SAMPLES, sampleTrail, TRACED_SAMPLES } from '../../lib/trace' import { FeedPanel } from './flow/FeedPanel' import { NodePanel } from './flow/NodePanel' import { SamplesTray } from './flow/SamplesTray' @@ -159,6 +159,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [capturing, setCapturing] = useState(false) const [nonce, setNonce] = useState(0) const [trayOpen, setTrayOpen] = useState(true) + // A traced sample followed through the flow (from 0). + const [followed, setFollowed] = useState(null) // A source from before flows has no places for its steps: they are laid out once, as loaded. const adopt = useCallback((r: SourceRow) => { @@ -209,6 +211,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o if (spec) edit(withFlow(spec, f)) } + const trail = useMemo(() => (followed === null ? null : sampleTrail(live.preview?.trace, followed)), [followed, live.preview]) + const nodes: FlowCanvasNode[] = useMemo(() => { if (!spec) return [] return flow.nodes.map((n) => ({ @@ -224,9 +228,10 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o summary: nodeSummary(spec, n), problems: byNode.byNode[n.id] ?? [], counts: live.preview?.steps?.[n.id], + trail: trail ? (trail[n.id] ?? 'absent') : undefined, } satisfies StepView, })) - }, [spec, flow, current, byNode, live.preview]) + }, [spec, flow, current, byNode, live.preview, trail]) const edges: FlowCanvasEdge[] = useMemo( () => @@ -234,9 +239,19 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const eid = `${e.source}->${e.target}` // What crossed it in the dry run: what its target took in. const crossed = live.preview?.steps?.[e.target]?.in - return { id: eid, source: e.source, target: e.target, selected: selectedEdges.has(eid), label: crossed === undefined ? undefined : String(crossed) } + // A followed sample's way through: from a step it passed to one it reached. + const taken = trail ? trail[e.source]?.state === 'passed' && !!trail[e.target] : undefined + return { + id: eid, + source: e.source, + target: e.target, + selected: selectedEdges.has(eid), + label: crossed === undefined ? undefined : String(crossed), + animated: taken, + className: taken === undefined ? undefined : taken ? 'flow-edge-taken' : 'flow-edge-faded', + } }), - [flow, selectedEdges, live.preview], + [flow, selectedEdges, live.preview, trail], ) const transportType = spec?.transport.type @@ -477,6 +492,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o error={live.error} open={trayOpen} onOpenChange={setTrayOpen} + followed={followed} + onFollow={setFollowed} /> {/* A step's settings (or the feed's) open over the right of the page, resizable. */} void + /** The traced sample being followed through the flow (from 0), if any. */ + followed: number | null + onFollow: (sample: number | null) => void }) { const [view, setView] = useState<'table' | 'json'>('table') - const [selected, setSelected] = useState(null) const parent = path.length > 1 ? path[path.length - 2] : undefined const rows = node && preview ? sampleRows(preview.trace, node.id, parent, path) : [] const counts = countsLine(preview) + // A row's sample: `2` or `2-0` (the first of what one sample made). + const sampleOf = (rowId: string) => Number(rowId.split('-')[0]) + const selected = followed === null ? null : (rows.find((r) => sampleOf(r.id) === followed)?.id ?? null) return ( + {followed !== null && ( + onFollow(null)} removeLabel="Stop following"> + Following #{followed + 1} + + )} A dry run of the source as edited (not yet saved) over its stored samples, as they leave the selected step. In the table, - fields the step added or changed are tinted and ones it removed are struck through. Nothing is published. + fields the step added or changed are tinted and ones it removed are struck through. Click a sample to follow it through the + flow: the steps it passed light up, and the one that dropped it says why. Nothing is published. onFollow(sampleOf(id) === followed ? null : sampleOf(id))} empty="No stored samples. Capture some on the input." /> ) : ( diff --git a/ui/src/pages/sources/flow/StepNode.tsx b/ui/src/pages/sources/flow/StepNode.tsx index 57e1006..54fd080 100644 --- a/ui/src/pages/sources/flow/StepNode.tsx +++ b/ui/src/pages/sources/flow/StepNode.tsx @@ -11,6 +11,8 @@ export interface StepView extends Record { problems: string[] /** What the dry run passed through it: taken in and passed on. */ counts?: { in: number; out: number } + /** While a sample is followed: whether it passed this step, was dropped or held here, or never came. */ + trail?: { state: 'passed' | 'dropped' | 'held'; note?: string } | 'absent' } /** @@ -23,13 +25,24 @@ export function StepNode({ data, selected, isConnectable }: NodeProps) { const failing = step.problems.length > 0 // A step that takes samples and passes none on is worth a look (a Filter that drops everything). const stops = !!step.counts && step.counts.in > 0 && step.counts.out === 0 + const mark = typeof step.trail === 'object' ? step.trail : undefined + const border = mark?.state === 'dropped' + ? 'var(--color-destructive)' + : mark + ? 'var(--color-accent)' + : failing + ? 'var(--color-destructive)' + : selected + ? 'var(--color-accent)' + : stops + ? 'var(--status-warning)' + : 'var(--border-strong)' return (
{takes(step.type) && }
@@ -40,6 +53,12 @@ export function StepNode({ data, selected, isConnectable }: NodeProps) { {KIND_WORD[step.type]}
{step.summary && {step.summary}} + {mark?.note && ( + + {mark.state === 'dropped' ? 'Dropped here: ' : 'Held here: '} + {mark.note} + + )} {step.counts && ( {step.counts.in} → {step.counts.out} From 683b638579c20b8dc314cff0cb87843a3e64b3c0 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Tue, 27 Oct 2026 19:51:16 -0500 Subject: [PATCH 6/6] UI: stareSDK 0.2.33 (#104) Tray and RecordTable (0.2.32), RecordTable draggableColumns (0.2.33). Co-Authored-By: Claude Opus 5.5 --- ui/package-lock.json | 8 ++++---- ui/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 08d4ba2..caf6d2e 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -23,7 +23,7 @@ "@dnd-kit/utilities": "^3.2.2", "@fontsource/montserrat": "^5.3.0", "@fontsource/oswald": "^5.3.0", - "@kineticlogic/staresdk": "^0.2.26", + "@kineticlogic/staresdk": "^0.2.33", "@lezer/highlight": "^1.2.4", "@xyflow/react": "^12.12.0", "framer-motion": "^12.43.0", @@ -269,9 +269,9 @@ "license": "MIT" }, "node_modules/@kineticlogic/staresdk": { - "version": "0.2.26", - "resolved": "https://registry.npmjs.org/@kineticlogic/staresdk/-/staresdk-0.2.26.tgz", - "integrity": "sha512-iPw0k72j+D7tAb0ZV2nsIxjYWWMZBSS+a4k28wNgSO/+6X8oKkSM/Bm9GdrnAaPFMs0InunBLA0rmcZSlOfh6A==", + "version": "0.2.33", + "resolved": "https://registry.npmjs.org/@kineticlogic/staresdk/-/staresdk-0.2.33.tgz", + "integrity": "sha512-/dyevW1nS7emSWLaboSNjgs/kTEXQdHywPvZnythM7nY4f0au6KXurbjKSwE7sCYXlQ9rQQsPLUJkbZHabB7tQ==", "license": "MIT", "engines": { "node": ">=20" diff --git a/ui/package.json b/ui/package.json index 2f41e33..04557d0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -26,7 +26,7 @@ "@dnd-kit/utilities": "^3.2.2", "@fontsource/montserrat": "^5.3.0", "@fontsource/oswald": "^5.3.0", - "@kineticlogic/staresdk": "^0.2.26", + "@kineticlogic/staresdk": "^0.2.33", "@lezer/highlight": "^1.2.4", "@xyflow/react": "^12.12.0", "framer-motion": "^12.43.0",