Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions ui/src/index.css
Original file line number Diff line number Diff line change
Expand Up @@ -970,6 +970,34 @@ body {
color: var(--status-warning);
}

/* Following a sample: the steps it never reached fade, the step that dropped it says why. */
.flow-step[data-trail='absent'] {
opacity: 0.4;
}

.flow-step-trail {
font-size: 10px;
line-height: 1.3;
color: var(--color-destructive);
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
overflow: hidden;
}

.flow-step-trail[data-state='held'] {
color: var(--status-warning);
}

.react-flow__edge.flow-edge-taken .react-flow__edge-path {
stroke: var(--color-accent);
stroke-width: 2;
}

.react-flow__edge.flow-edge-faded {
opacity: 0.3;
}

.source-editor-json {
flex: 1;
min-height: 0;
Expand Down
30 changes: 29 additions & 1 deletion ui/src/lib/trace.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import type { PreviewTrace } from '../api/client'
import { countsLine, sampleRows, samplesAt } from './trace'
import { countsLine, sampleRows, sampleTrail, samplesAt } from './trace'

// Frame 0 decodes to three records (samples #1–#3), frame 1 does not decode (#4), frame 2 is
// binary (#5), frame 3 gives #6.
Expand Down Expand Up @@ -170,3 +170,31 @@ describe('sampleRows', () => {
expect(rows.map((r) => r.value ?? r.dropped)).toEqual([{ n: 1 }, { n: 2 }, 'decode error: bad'])
})
})

describe('sampleTrail', () => {
it('marks the steps a sample passed and the one that dropped it', () => {
const trace = {
frames: [],
samples: [
{
frame: 0,
stages: [
{ id: 'input', items: [{ n: 1 }] },
{ id: 'map', items: [{ k: 1 }] },
{ id: 'f', items: [], dropped: ['drop_if matched'] },
{ id: 'raw', items: [{ k: 1 }] },
{ id: 'out', items: [] },
{ id: 'publish', items: [] },
],
},
],
}
expect(sampleTrail(trace, 0)).toEqual({
input: { state: 'passed' },
map: { state: 'passed' },
f: { state: 'dropped', note: 'drop_if matched' },
raw: { state: 'passed' },
})
expect(sampleTrail(trace, 5)).toEqual({})
})
})
16 changes: 16 additions & 0 deletions ui/src/lib/trace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,22 @@ export function sampleRows(trace: PreviewTrace | undefined, stageId: string, par
})
}

/** Where one traced sample went: each step it passed, the one that dropped or held it, and why. */
export type TrailMark = { state: 'passed' | 'dropped' | 'held'; note?: string }

/** The trail of traced sample `index` (as `sampleRows` numbers them, from 0), by step id. */
export function sampleTrail(trace: PreviewTrace | undefined, index: number): Record<string, TrailMark> {
const s = trace?.samples[index]
const out: Record<string, TrailMark> = {}
for (const x of s?.stages ?? []) {
if (x.id === 'publish') continue
if (x.dropped?.length) out[x.id] = { state: 'dropped', note: x.dropped.join(' · ') }
else if (x.held) out[x.id] = { state: 'held', note: 'waiting for the rest of its scan' }
else if (x.items.length) out[x.id] = { state: 'passed' }
}
return out
}

/** The counts on one line, as `frames 5 · records 5 · emitted 4`. */
export function countsLine(result: PreviewResult | null): string {
const counts = Object.entries(result?.counts ?? {}).filter(([k]) => !k.startsWith('rejected:') && !k.startsWith('grade:'))
Expand Down
25 changes: 21 additions & 4 deletions ui/src/pages/sources/SourceEditor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { useCan } from '../../auth/context'
import { InfoTip } from '../../components/InfoTip'
import { errorMessage } from '../../lib/format'
import { SOURCE_STATES_INFO, sourceState } from '../../lib/sourceState'
import { MAX_TRACED_SAMPLES, TRACED_SAMPLES } from '../../lib/trace'
import { MAX_TRACED_SAMPLES, sampleTrail, TRACED_SAMPLES } from '../../lib/trace'
import { FeedPanel } from './flow/FeedPanel'
import { NodePanel } from './flow/NodePanel'
import { SamplesTray } from './flow/SamplesTray'
Expand Down Expand Up @@ -159,6 +159,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
const [capturing, setCapturing] = useState(false)
const [nonce, setNonce] = useState(0)
const [trayOpen, setTrayOpen] = useState(true)
// A traced sample followed through the flow (from 0).
const [followed, setFollowed] = useState<number | null>(null)

// A source from before flows has no places for its steps: they are laid out once, as loaded.
const adopt = useCallback((r: SourceRow) => {
Expand Down Expand Up @@ -209,6 +211,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
if (spec) edit(withFlow(spec, f))
}

const trail = useMemo(() => (followed === null ? null : sampleTrail(live.preview?.trace, followed)), [followed, live.preview])

const nodes: FlowCanvasNode[] = useMemo(() => {
if (!spec) return []
return flow.nodes.map((n) => ({
Expand All @@ -224,19 +228,30 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
summary: nodeSummary(spec, n),
problems: byNode.byNode[n.id] ?? [],
counts: live.preview?.steps?.[n.id],
trail: trail ? (trail[n.id] ?? 'absent') : undefined,
} satisfies StepView,
}))
}, [spec, flow, current, byNode, live.preview])
}, [spec, flow, current, byNode, live.preview, trail])

const edges: FlowCanvasEdge[] = useMemo(
() =>
flow.edges.map((e) => {
const eid = `${e.source}->${e.target}`
// What crossed it in the dry run: what its target took in.
const crossed = live.preview?.steps?.[e.target]?.in
return { id: eid, source: e.source, target: e.target, selected: selectedEdges.has(eid), label: crossed === undefined ? undefined : String(crossed) }
// A followed sample's way through: from a step it passed to one it reached.
const taken = trail ? trail[e.source]?.state === 'passed' && !!trail[e.target] : undefined
return {
id: eid,
source: e.source,
target: e.target,
selected: selectedEdges.has(eid),
label: crossed === undefined ? undefined : String(crossed),
animated: taken,
className: taken === undefined ? undefined : taken ? 'flow-edge-taken' : 'flow-edge-faded',
}
}),
[flow, selectedEdges, live.preview],
[flow, selectedEdges, live.preview, trail],
)

const transportType = spec?.transport.type
Expand Down Expand Up @@ -477,6 +492,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
error={live.error}
open={trayOpen}
onOpenChange={setTrayOpen}
followed={followed}
onFollow={setFollowed}
/>
{/* A step's settings (or the feed's) open over the right of the page, resizable. */}
<SideNav
Expand Down
21 changes: 17 additions & 4 deletions ui/src/pages/sources/flow/SamplesTray.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { useState } from 'react'
import { RecordTable, SegmentedControl, Tray } from '@kineticlogic/staresdk'
import { Badge, RecordTable, SegmentedControl, Tray } from '@kineticlogic/staresdk'
import type { PreviewResult } from '../../../api/client'
import { InfoTip } from '../../../components/InfoTip'
import { PIPELINE_COUNTS_INFO } from '../../../lib/sourceState'
Expand All @@ -23,6 +23,8 @@ export function SamplesTray({
error,
open,
onOpenChange,
followed,
onFollow,
}: {
node: FlowNode | null
title: string
Expand All @@ -32,12 +34,17 @@ export function SamplesTray({
error: string | null
open: boolean
onOpenChange: (open: boolean) => void
/** The traced sample being followed through the flow (from 0), if any. */
followed: number | null
onFollow: (sample: number | null) => void
}) {
const [view, setView] = useState<'table' | 'json'>('table')
const [selected, setSelected] = useState<string | null>(null)
const parent = path.length > 1 ? path[path.length - 2] : undefined
const rows = node && preview ? sampleRows(preview.trace, node.id, parent, path) : []
const counts = countsLine(preview)
// A row's sample: `2` or `2-0` (the first of what one sample made).
const sampleOf = (rowId: string) => Number(rowId.split('-')[0])
const selected = followed === null ? null : (rows.find((r) => sampleOf(r.id) === followed)?.id ?? null)

return (
<Tray
Expand All @@ -51,9 +58,15 @@ export function SamplesTray({
storageKey="ot.sources.samples.height"
actions={
<>
{followed !== null && (
<Badge size="sm" color="brand" onRemove={() => onFollow(null)} removeLabel="Stop following">
Following #{followed + 1}
</Badge>
)}
<InfoTip label="Samples">
A dry run of the source as edited (not yet saved) over its stored samples, as they leave the selected step. In the table,
fields the step added or changed are tinted and ones it removed are struck through. Nothing is published.
fields the step added or changed are tinted and ones it removed are struck through. Click a sample to follow it through the
flow: the steps it passed light up, and the one that dropped it says why. Nothing is published.
</InfoTip>
<SegmentedControl
aria-label="Samples view"
Expand Down Expand Up @@ -89,7 +102,7 @@ export function SamplesTray({
// A Map makes observations out of records: everything would differ.
showChanges={node.type !== 'input' && node.type !== 'map'}
selectedId={selected}
onRowClick={setSelected}
onRowClick={(id) => onFollow(sampleOf(id) === followed ? null : sampleOf(id))}
empty="No stored samples. Capture some on the input."
/>
) : (
Expand Down
25 changes: 22 additions & 3 deletions ui/src/pages/sources/flow/StepNode.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ export interface StepView extends Record<string, unknown> {
problems: string[]
/** What the dry run passed through it: taken in and passed on. */
counts?: { in: number; out: number }
/** While a sample is followed: whether it passed this step, was dropped or held here, or never came. */
trail?: { state: 'passed' | 'dropped' | 'held'; note?: string } | 'absent'
}

/**
Expand All @@ -23,13 +25,24 @@ export function StepNode({ data, selected, isConnectable }: NodeProps) {
const failing = step.problems.length > 0
// A step that takes samples and passes none on is worth a look (a Filter that drops everything).
const stops = !!step.counts && step.counts.in > 0 && step.counts.out === 0
const mark = typeof step.trail === 'object' ? step.trail : undefined
const border = mark?.state === 'dropped'
? 'var(--color-destructive)'
: mark
? 'var(--color-accent)'
: failing
? 'var(--color-destructive)'
: selected
? 'var(--color-accent)'
: stops
? 'var(--status-warning)'
: 'var(--border-strong)'
return (
<div
className="flow-step"
data-failing={failing || undefined}
style={{
border: `1px solid ${failing ? 'var(--color-destructive)' : selected ? 'var(--color-accent)' : stops ? 'var(--status-warning)' : 'var(--border-strong)'}`,
}}
data-trail={mark?.state ?? (step.trail === 'absent' ? 'absent' : undefined)}
style={{ border: `1px solid ${border}` }}
>
{takes(step.type) && <Handle type="target" position={Position.Left} isConnectable={isConnectable} />}
<div className="flow-step-head">
Expand All @@ -40,6 +53,12 @@ export function StepNode({ data, selected, isConnectable }: NodeProps) {
<span className="flow-step-kind">{KIND_WORD[step.type]}</span>
</div>
{step.summary && <span className="flow-step-summary">{step.summary}</span>}
{mark?.note && (
<span className="flow-step-trail" data-state={mark.state}>
{mark.state === 'dropped' ? 'Dropped here: ' : 'Held here: '}
{mark.note}
</span>
)}
{step.counts && (
<span className="flow-step-counts" data-stops={stops || undefined} title={`${step.counts.in} in, ${step.counts.out} out over the stored samples`}>
{step.counts.in} → {step.counts.out}
Expand Down
Loading