From 40fff21b614ad0318c7d28de6a13c7e136c39d73 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Wed, 2 Sep 2026 07:17:28 -0400 Subject: [PATCH] =?UTF-8?q?feat(lifeboard):=20provenance=20=E2=80=94=20eve?= =?UTF-8?q?ry=20fact=20can=20say=20where=20it=20came=20from?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answered from the log rather than from a side table. The log already records, for every change, what it was derived from, who made it and what it replaced -- recorded at the moment the fact was created, because nothing else was possible. That is what makes provenance work at all; a derivation bolted on afterwards is a guess, and a guess about where a vaccination date came from is worse than nothing. Derivation records add only the two things the log cannot infer: which kind of step it was, and how confident a model was. A source that has changed is flagged, never silently re-read. Re-running an extraction against a changed document behind someone's back is how a date quietly becomes a different date. Lineage is drawn as a flow board because the thing people need is the shape -- that a date came from one email through one extraction, or from three documents through a merge and a correction. A list of steps reads as bureaucracy. Deep links live in a table as data. Every connector addresses a document differently and the formats change without warning; a broken link should be a one-line fix. Drive gets the page number, Gmail opens the thread. Boards reach the rest of the app through an href prefix the shell intercepts. A board is data and cannot carry a click handler -- giving it one would mean the agent authors behaviour rather than a picture. Anything else in an href is an ordinary link and leaves the app, which is what a source link should do. Pairs a derivation to the entry it explains by sequence number, not timestamp: a test caught two unrelated writes landing in the same millisecond, which mislabelled the history of the fact someone was trying to check. Co-Authored-By: Claude Opus 5 (1M context) --- .../lifeboard/src/connectors/deeplinks.ts | 62 +++++ packages/lifeboard/src/provenance/graph.ts | 121 +++++++++ packages/lifeboard/src/provenance/record.ts | 115 +++++++++ packages/lifeboard/src/provenance/types.ts | 61 +++++ packages/lifeboard/src/rail.ts | 1 + packages/lifeboard/src/shell.ts | 14 ++ packages/lifeboard/src/views/focused.ts | 6 +- packages/lifeboard/src/views/lineage.ts | 59 +++++ packages/lifeboard/src/views/links.ts | 23 ++ packages/lifeboard/src/views/sources.ts | 91 +++++++ packages/lifeboard/src/views/types.ts | 6 +- packages/lifeboard/test/provenance.test.ts | 230 ++++++++++++++++++ packages/lifeboard/test/shell.test.ts | 30 ++- 13 files changed, 815 insertions(+), 4 deletions(-) create mode 100644 packages/lifeboard/src/connectors/deeplinks.ts create mode 100644 packages/lifeboard/src/provenance/graph.ts create mode 100644 packages/lifeboard/src/provenance/record.ts create mode 100644 packages/lifeboard/src/provenance/types.ts create mode 100644 packages/lifeboard/src/views/lineage.ts create mode 100644 packages/lifeboard/src/views/links.ts create mode 100644 packages/lifeboard/src/views/sources.ts create mode 100644 packages/lifeboard/test/provenance.test.ts diff --git a/packages/lifeboard/src/connectors/deeplinks.ts b/packages/lifeboard/src/connectors/deeplinks.ts new file mode 100644 index 0000000..96606e0 --- /dev/null +++ b/packages/lifeboard/src/connectors/deeplinks.ts @@ -0,0 +1,62 @@ +// Deep links, as data. Every connector addresses a document differently and the formats change +// without warning, so they live in a table rather than in code paths — a broken link is then a +// one-line fix, not a hunt through three modules. +// +// The locator is the part inside the document: a page in a PDF, a paragraph in an email. Where a +// connector supports addressing it, the template uses it; where it does not, the link opens the +// document and the excerpt on the lineage card tells you what to look for. + +export type ConnectorId = "gmail" | "gdrive" | "dropbox" | "photo" | "web" | "calendar" | "manual"; + +export interface LinkTemplate { + /** {id} is the document, {locator} the place inside it, {query} a search fallback. */ + document: string; + withLocator?: string; + label: string; +} + +export const LINKS: Record = { + // A message id addresses the thread; rfc822msgid search is the fallback when only the header id + // is known, which is what an IMAP-shaped extraction usually has. + gmail: { + document: "https://mail.google.com/mail/u/0/#all/{id}", + label: "Open in Gmail", + }, + gdrive: { + document: "https://drive.google.com/file/d/{id}/view", + withLocator: "https://drive.google.com/file/d/{id}/view#page={locator}", + label: "Open in Drive", + }, + dropbox: { + document: "https://www.dropbox.com/home?preview={id}", + label: "Open in Dropbox", + }, + photo: { document: "{id}", label: "Open the photo" }, + web: { document: "{id}", label: "Open the page" }, + calendar: { + document: "https://calendar.google.com/calendar/u/0/r/eventedit/{id}", + label: "Open in Calendar", + }, + manual: { document: "", label: "Entered by hand" }, +}; + +export interface LinkTarget { + connector: ConnectorId; + id: string; + locator?: string; +} + +/** The URL to open, or "" when the source is not something that can be opened. */ +export function deepLink(target: LinkTarget): string { + const t = LINKS[target.connector]; + if (!t) return ""; + const template = target.locator && t.withLocator ? t.withLocator : t.document; + return template.replace("{id}", encodeId(target.connector, target.id)).replace("{locator}", encodeURIComponent(target.locator ?? "")); +} + +export const linkLabel = (connector: ConnectorId): string => LINKS[connector]?.label ?? "Open"; + +/** Photos and web pages are already URLs; ids inside a provider's namespace need escaping. */ +function encodeId(connector: ConnectorId, id: string): string { + return connector === "photo" || connector === "web" ? id : encodeURIComponent(id); +} diff --git a/packages/lifeboard/src/provenance/graph.ts b/packages/lifeboard/src/provenance/graph.ts new file mode 100644 index 0000000..17ab68b --- /dev/null +++ b/packages/lifeboard/src/provenance/graph.ts @@ -0,0 +1,121 @@ +// "Where did this come from?" answered from the log rather than from a side table. +// +// The log already records, for every change: what it was derived from (`from`), who made it (`by`), +// and what it replaced. That is most of a derivation graph, recorded at the moment the fact was +// created because nothing else was possible — which is the only way provenance ever works. A +// `derivation` record adds the two things the log cannot infer: which KIND of step it was, and how +// confident a model was. +// +// One rule runs through this file: a source that has changed is FLAGGED, never silently re-read. +// Re-deriving from a changed document behind someone's back is how a vaccination date quietly +// becomes a different date. + +import type { Store } from "../store.js"; +import { deepLink, linkLabel } from "../connectors/deeplinks.js"; +import type { Entry } from "../types.js"; +import { parseRef } from "../types.js"; +import type { Derivation, Lineage, LineageEdge, LineageNode, Source, Step } from "./types.js"; + +const MAX_DEPTH = 6; + +export function lineageOf(store: Store, factRef: string): Lineage { + const nodes = new Map(); + const edges: LineageEdge[] = []; + walk(store, factRef, nodes, edges, 0, new Set()); + return { nodes: [...nodes.values()], edges }; +} + +function walk(store: Store, ref: string, nodes: Map, edges: LineageEdge[], depth: number, seen: Set): void { + if (depth > MAX_DEPTH || seen.has(ref)) return; + seen.add(ref); + + const { collection, id } = parseRef(ref); + if (collection === "source") { + addSource(store, id, nodes); + return; + } + + const history = store.history(ref); + if (history.length === 0) return; + + const derivations = store.list("derivation", { match: { factRef: ref } }); + const current = history[history.length - 1]; + nodes.set(ref, { + id: ref, + kind: "fact", + label: describeValue(current), + detail: `${collection} · ${current.op === "retract" ? "removed" : "as it stands now"}`, + }); + + // Every entry is a step. Corrections keep the value they replaced visible: the point of a lineage + // view is to show that a number changed, not to present the latest one as if it always was. + for (const entry of history) { + const stepId = `${ref}#${entry.seq}`; + const derivation = derivations.find((d) => d.entrySeq === entry.seq); + nodes.set(stepId, { + id: stepId, + kind: "step", + label: stepLabel(entry, derivation?.step), + detail: [author(entry), derivation?.confidence !== undefined ? `${Math.round(derivation.confidence * 100)}% confident` : "", derivation?.note ?? ""].filter(Boolean).join(" · "), + ...(entry.op === "correct" || entry !== current ? { superseded: entry !== current } : {}), + }); + edges.push({ id: `${stepId}->${ref}`, source: stepId, target: ref }); + + for (const input of (entry.op === "retract" ? [] : entry.from) ?? []) { + edges.push({ id: `${input}->${stepId}`, source: input, target: stepId, label: "from" }); + walk(store, input, nodes, edges, depth + 1, seen); + } + } +} + +function addSource(store: Store, id: string, nodes: Map): void { + const source = store.get("source", id); + const ref = `source/${id}`; + if (!source) { + nodes.set(ref, { id: ref, kind: "source", label: "A source that is no longer here" }); + return; + } + nodes.set(ref, { + id: ref, + kind: "source", + label: source.title, + detail: [source.excerpt, source.locator ? `page ${source.locator}` : "", linkLabel(source.connector)].filter(Boolean).join(" · "), + url: deepLink({ connector: source.connector, id: source.ref, locator: source.locator }), + ...(source.stale ? { stale: true } : {}), + }); +} + +/** SHA-256 of the document's text. Used only to notice a change; nothing re-reads on its own. */ +export async function contentHash(text: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + +/** + * Has the document behind this source changed? Answering yes marks the facts drawn from it as + * needing a look — it never re-extracts them. A model-derived fact especially: re-running the + * extraction could change a date nobody asked to change. + */ +export async function sourceChanged(source: Source, currentText: string): Promise { + if (!source.contentHash) return false; + return (await contentHash(currentText)) !== source.contentHash; +} + +const author = (e: Entry): string => + e.by.kind === "human" ? `by ${e.by.profile}` : e.by.kind === "code" ? `computed by ${e.by.fn}` : `from ${e.by.model}`; + +function stepLabel(entry: Entry, step?: Step): string { + if (step) return step; + if (entry.op === "assert") return entry.by.kind === "human" ? "entered" : entry.by.kind === "code" ? "computed" : "extracted"; + if (entry.op === "correct") return "corrected"; + return "removed"; +} + +function describeValue(entry: Entry): string { + if (entry.op === "retract") return "(removed)"; + const v = entry.value as Record; + for (const key of ["text", "title", "name", "summary"]) { + if (typeof v[key] === "string" && v[key]) return v[key] as string; + } + return parseRef(entry.ref).id; +} diff --git a/packages/lifeboard/src/provenance/record.ts b/packages/lifeboard/src/provenance/record.ts new file mode 100644 index 0000000..12640ce --- /dev/null +++ b/packages/lifeboard/src/provenance/record.ts @@ -0,0 +1,115 @@ +// The producer side of provenance: helpers every path that creates a fact uses, so the graph is +// recorded at the moment the fact is created. Threading this through the producers is the whole +// job — a derivation bolted on afterwards is a guess, and a guess about where a vaccination date +// came from is worse than nothing. + +import type { Store } from "../store.js"; +import { add, update, type By, type CollectionName, type Mutation } from "../types.js"; +import { contentHash } from "./graph.js"; +import type { Derivation, Source, Step } from "./types.js"; +import type { ConnectorId } from "../connectors/deeplinks.js"; + +export interface RecordSourceInput { + space: string; + connector: ConnectorId; + title: string; + ref: string; + locator?: string; + excerpt?: string; + /** The document's text, hashed so a later change can be noticed. */ + text?: string; + retrievedAt?: string; +} + +/** Register a document. Returns the mutation and the ref facts should point `from` at. */ +export async function sourceMutation(input: RecordSourceInput, id: string, now = new Date()): Promise<{ mutation: Mutation; ref: string }> { + const value: Omit = { + id, + space: input.space, + connector: input.connector, + title: input.title, + ref: input.ref, + retrievedAt: input.retrievedAt ?? now.toISOString(), + ...(input.locator ? { locator: input.locator } : {}), + ...(input.excerpt ? { excerpt: input.excerpt } : {}), + ...(input.text ? { contentHash: await contentHash(input.text) } : {}), + }; + return { mutation: add("source", value as unknown as Parameters[1]), ref: `source/${id}` }; +} + +export interface DerivationInput { + space: string; + factRef: string; + /** The log entry this explains. */ + entrySeq: number; + step: Step; + inputs: string[]; + by: Derivation["by"]; + confidence?: number; + note?: string; + at?: string; +} + +export const derivationMutation = (input: DerivationInput, now = new Date()): Mutation => + add("derivation", { + space: input.space, + factRef: input.factRef, + entrySeq: input.entrySeq, + step: input.step, + inputs: input.inputs, + by: input.by, + at: input.at ?? now.toISOString(), + ...(input.confidence !== undefined ? { confidence: input.confidence } : {}), + ...(input.note ? { note: input.note } : {}), + }); + +/** + * Change a fact by hand. The correction and the value it replaced both stay in the log, and the + * correction is marked as human so nothing later re-derives over the top of it. + */ +export async function correct( + store: Store, + factRef: string, + patch: Record, + who: By, + space = "shared", +): Promise { + const [collection, ...rest] = factRef.split("/"); + const id = rest.join("/"); + if (!store.get(collection as CollectionName, id)) return false; + const entries = await store.apply([update(collection as CollectionName, id, patch)], who); + if (entries.length === 0) return false; + await store.apply( + [derivationMutation({ + space, + factRef, + entrySeq: entries[0].seq, + step: "corrected", + inputs: [], + by: who.kind === "human" ? "human" : who.kind === "code" ? "code" : "model", + note: "Corrected by hand.", + })], + who, + ); + return true; +} + +/** + * Mark every fact drawn from a source as needing a look. Flagging, never re-reading: re-running an + * extraction against a changed document could alter a value nobody asked to alter. + */ +export async function flagStale(store: Store, sourceRef: string, by: By): Promise { + const touched = new Set(); + for (const entry of store.history()) { + if (entry.op === "retract" || !entry.from?.includes(sourceRef)) continue; + touched.add(entry.ref); + } + const { id } = { id: sourceRef.slice(sourceRef.indexOf("/") + 1) }; + const mutations: Mutation[] = [update("source", id, { stale: true })]; + for (const ref of touched) { + const [collection, ...rest] = ref.split("/"); + mutations.push(update(collection as CollectionName, rest.join("/"), { needsReview: true })); + } + await store.apply(mutations, by); + return [...touched]; +} diff --git a/packages/lifeboard/src/provenance/types.ts b/packages/lifeboard/src/provenance/types.ts new file mode 100644 index 0000000..899372c --- /dev/null +++ b/packages/lifeboard/src/provenance/types.ts @@ -0,0 +1,61 @@ +import type { Base } from "../types.js"; +import type { ConnectorId } from "../connectors/deeplinks.js"; + +/** A document a fact came from. Recorded when the fact is created, never reconstructed later. */ +export interface Source extends Base { + connector: ConnectorId; + title: string; + /** The provider's id for the document — the deep link is built from it, not stored pre-built, + * so a change in a provider's URL format fixes every old source at once. */ + ref: string; + /** Where inside the document: a page number, a paragraph index. */ + locator?: string; + excerpt?: string; + retrievedAt: string; + /** For noticing that the document changed. Compared, never used to re-read it automatically. */ + contentHash?: string; + /** Set once the document is known to have changed. The facts drawn from it are flagged too. */ + stale?: boolean; +} + +export type Step = "extracted" | "parsed" | "scaled" | "merged" | "computed" | "entered" | "corrected"; + +/** How one fact came from others. The log's own `from` and `by` carry most of this; a derivation + * record adds the step and the confidence, which the log cannot infer. */ +export interface Derivation extends Base { + factRef: string; + /** The sequence number of the log entry this explains. Identity, not a timestamp: two things can + * happen in the same millisecond, and pairing a derivation to the wrong entry mislabels the + * history of a fact someone is trying to check. */ + entrySeq: number; + step: Step; + inputs: string[]; + by: "model" | "code" | "human"; + confidence?: number; + note?: string; + at: string; +} + +export interface LineageNode { + id: string; + kind: "fact" | "step" | "source"; + label: string; + detail?: string; + /** Present on source nodes: where tapping takes you. */ + url?: string; + /** True when the underlying document has changed since the fact was taken from it. */ + stale?: boolean; + superseded?: boolean; +} + +export interface LineageEdge { + id: string; + source: string; + target: string; + label?: string; +} + +export interface Lineage { + nodes: LineageNode[]; + edges: LineageEdge[]; +} diff --git a/packages/lifeboard/src/rail.ts b/packages/lifeboard/src/rail.ts index bcbf272..09545b3 100644 --- a/packages/lifeboard/src/rail.ts +++ b/packages/lifeboard/src/rail.ts @@ -15,6 +15,7 @@ export const DEFAULT_RAIL: RailItem[] = [ { label: "Today", route: { mode: "focused", view: "today" } }, { label: "Meals", route: { mode: "focused", view: "meals" } }, { label: "Shopping", route: { mode: "focused", view: "groceries" } }, + { label: "Sources", route: { mode: "focused", view: "sources" } }, { label: "Connect", route: { mode: "focused", view: "connections" } }, ]; diff --git a/packages/lifeboard/src/shell.ts b/packages/lifeboard/src/shell.ts index c7c7711..ed4f920 100644 --- a/packages/lifeboard/src/shell.ts +++ b/packages/lifeboard/src/shell.ts @@ -24,6 +24,7 @@ import { add, update, type By, type CollectionName, type Entry, type Item, type import { renderConnections } from "./views/connections.js"; import { buildFocusedBoard } from "./views/focused.js"; import { buildGridBoard } from "./views/grid.js"; +import { routeFromHref } from "./views/links.js"; import { parseRoute, routeKey, type Board, type BoardContext, type Route } from "./views/types.js"; import { parseRef } from "./types.js"; @@ -319,6 +320,18 @@ export async function startShell(deps: ShellDeps): Promise { } }); + // A board cannot carry a click handler — it is data, and letting the agent author behaviour is a + // different product. An anchor with a known prefix is the whole mechanism; everything else in an + // href is an ordinary link and is left alone, which is what a link out to Gmail needs. + const onBoardClick = (e: Event) => { + const anchor = (e.target as HTMLElement | null)?.closest?.("a"); + const route = routeFromHref(anchor?.getAttribute("href")); + if (!route) return; + e.preventDefault(); + void shell.navigate(route); + }; + regions.board.addEventListener("click", onBoardClick); + const offStore = store.subscribe((changed) => { // Console chatter is stored in the same log; it must not cause the board to redraw. if (changed.size === 1 && changed.has("message")) return; @@ -384,6 +397,7 @@ export async function startShell(deps: ShellDeps): Promise { destroy() { offStore(); offConsole(); + regions.board.removeEventListener("click", onBoardClick); }, }; diff --git a/packages/lifeboard/src/views/focused.ts b/packages/lifeboard/src/views/focused.ts index 9d7f7bb..92c0388 100644 --- a/packages/lifeboard/src/views/focused.ts +++ b/packages/lifeboard/src/views/focused.ts @@ -2,11 +2,13 @@ import { buildListBoard, buildTodayBoard } from "../boards.js"; import { buildMealPlanBoard, buildRecipeBoard, recipesIn } from "./recipe.js"; +import { buildLineageBoard } from "./lineage.js"; +import { buildSourcesBoard } from "./sources.js"; import type { Recipe } from "../recipes/types.js"; import type { Item } from "../types.js"; import type { Board, BoardContext } from "./types.js"; -const TITLES: Record = { groceries: "Groceries", todo: "To do", connections: "Connections", meals: "Meals" }; +const TITLES: Record = { groceries: "Groceries", todo: "To do", connections: "Connections", meals: "Meals", sources: "Where things came from" }; export interface StoredBoard { id: string; @@ -35,6 +37,8 @@ export function buildFocusedBoard(view: string, ctx: BoardContext): Board { if (recipe) return buildRecipeBoard(recipe); } if (view === "meals") return buildMealPlanBoard(recipesIn(ctx), ctx); + if (view === "sources") return buildSourcesBoard(ctx); + if (view.startsWith("lineage:")) return buildLineageBoard(ctx.store, view.slice("lineage:".length)); const items = ctx.store.list("item", { space: ctx.spaces }); diff --git a/packages/lifeboard/src/views/lineage.ts b/packages/lifeboard/src/views/lineage.ts new file mode 100644 index 0000000..15875ec --- /dev/null +++ b/packages/lifeboard/src/views/lineage.ts @@ -0,0 +1,59 @@ +// The lineage view: a flow board you can tap through to the document a fact came from. +// +// Drawn as `flow` rather than as a list because the thing people need to see is the SHAPE — that a +// date came from one email through one extraction, or from three documents through a merge and a +// correction. A list of steps reads as bureaucracy; a graph reads as an answer. + +import type { Store } from "../store.js"; +import { lineageOf } from "../provenance/graph.js"; +import type { Lineage } from "../provenance/types.js"; +import type { Board } from "./types.js"; + +const COLOURS = { + fact: "#d8b45a", + step: "#6f7280", + source: "#5b8def", + stale: "#e0725e", +} as const; + +export function buildLineageBoard(store: Store, factRef: string): Board { + const lineage = lineageOf(store, factRef); + if (lineage.nodes.length === 0) { + return { + type: "component", + title: "Where this came from", + content: JSON.stringify({ + type: "BoardHeader", + props: { title: "Nothing to trace", lede: "This fact is no longer here, or it never carried a source." }, + }), + }; + } + return { type: "flow", title: "Where this came from", content: JSON.stringify(toFlow(lineage)) }; +} + +export function toFlow(lineage: Lineage): { nodes: unknown[]; edges: unknown[]; title: string; legend?: unknown } { + return { + title: "Where this came from", + nodes: lineage.nodes.map((n) => ({ + id: n.id, + label: n.label, + ...(n.detail ? { sublabel: n.detail } : {}), + ...(n.url ? { url: n.url } : {}), + shape: n.kind === "source" ? "cylinder" : n.kind === "step" ? "hexagon" : "box", + color: n.stale ? COLOURS.stale : COLOURS[n.kind], + ...(n.superseded ? { opacity: 0.55 } : {}), + })), + edges: lineage.edges.map((e) => ({ + id: e.id, + source: e.source, + target: e.target, + ...(e.label ? { label: e.label } : {}), + })), + legend: [ + { color: COLOURS.source, label: "Document" }, + { color: COLOURS.step, label: "Step" }, + { color: COLOURS.fact, label: "The fact" }, + { color: COLOURS.stale, label: "Source has changed" }, + ], + }; +} diff --git a/packages/lifeboard/src/views/links.ts b/packages/lifeboard/src/views/links.ts new file mode 100644 index 0000000..d73221a --- /dev/null +++ b/packages/lifeboard/src/views/links.ts @@ -0,0 +1,23 @@ +// Links from inside a board back into the app. +// +// A board is data — it cannot carry a click handler, and giving it one would mean the agent could +// author behaviour rather than a picture. An anchor with a known href prefix is the whole +// mechanism: the shell intercepts it and navigates. Anything else in an href is an ordinary link +// and leaves the app, which is exactly what a source link should do. + +import type { Route } from "./types.js"; +import { parseRoute, routeKey } from "./types.js"; + +const PREFIX = "#lb:"; + +export const routeHref = (route: Route): string => `${PREFIX}${routeKey(route)}`; + +/** The route an href asks for, or null when it is an ordinary link. */ +export function routeFromHref(href: string | null | undefined): Route | null { + if (!href) return null; + const i = href.indexOf(PREFIX); + if (i < 0) return null; + const key = href.slice(i + PREFIX.length); + const route = parseRoute(key, { mode: "grid", view: "home" }); + return routeKey(route) === key ? route : null; +} diff --git a/packages/lifeboard/src/views/sources.ts b/packages/lifeboard/src/views/sources.ts new file mode 100644 index 0000000..8d1016d --- /dev/null +++ b/packages/lifeboard/src/views/sources.ts @@ -0,0 +1,91 @@ +// "Where did this come from?" needs a way in. This board lists every fact that carries a source, +// newest first, each linking to its own lineage — so the answer is two taps from anywhere rather +// than a feature you have to know exists. + +import type { Store } from "../store.js"; +import { parseRef } from "../types.js"; +import type { Board, BoardContext } from "./types.js"; +import { routeHref } from "./links.js"; + +interface Traced { + ref: string; + label: string; + at: string; + by: string; + needsReview?: boolean; +} + +export function tracedFacts(store: Store, spaces: string[]): Traced[] { + const seen = new Map(); + for (const entry of store.history()) { + // The entry that carries `from` is the one that says where the fact came from; the flag saying + // its source has since changed lands on a LATER entry, so the label and the flag are read from + // the record as it stands rather than from this entry's frozen value. + if (entry.op === "retract" || !entry.from?.length) continue; + const { collection, id } = parseRef(entry.ref); + const current = store.get>(collection, id); + if (!current || !spaces.includes(current.space as string)) continue; + seen.set(entry.ref, { + ref: entry.ref, + label: (["text", "title", "name"].map((k) => current[k]).find((x) => typeof x === "string" && x) as string) ?? id, + at: entry.at, + by: entry.by.kind === "human" ? `entered by ${entry.by.profile}` : entry.by.kind === "code" ? `computed by ${entry.by.fn}` : `read by ${entry.by.model}`, + ...(current.needsReview ? { needsReview: true } : {}), + }); + } + return [...seen.values()].reverse(); +} + +export function buildSourcesBoard(ctx: BoardContext): Board { + const facts = tracedFacts(ctx.store, ctx.spaces); + const flagged = facts.filter((f) => f.needsReview).length; + + return { + type: "component", + title: "Where things came from", + content: JSON.stringify({ + type: "Stack", + props: { gap: "sm" }, + children: [ + { + type: "BoardHeader", + props: { + title: "Where things came from", + lede: facts.length === 0 + ? "Nothing here was taken from a document yet. Anything the assistant reads out of an email or a PDF will be listed here with a way back to the original." + : `${facts.length} ${facts.length === 1 ? "thing was" : "things were"} taken from a document. Open one to see the chain back to the original.`, + ...(flagged ? { howToUse: `${flagged} of these came from a document that has since changed — they are marked, and nothing was re-read automatically.` } : {}), + }, + }, + ...(facts.length === 0 + ? [] + : [{ + type: "Stack", + props: { gap: "xs" }, + children: facts.slice(0, 50).map((f) => ({ + type: "Paper", + props: { withBorder: true, p: "sm" }, + children: [ + { + type: "Group", + props: { justify: "space-between", align: "center" }, + children: [ + { + type: "Stack", + props: { gap: 2 }, + children: [ + { type: "Text", props: { fw: 600, size: "sm" }, children: f.label }, + { type: "Text", props: { size: "xs", c: "dimmed" }, children: `${f.by} · ${f.at.slice(0, 10)}` }, + ], + }, + ...(f.needsReview ? [{ type: "Badge", props: { color: "orange", variant: "light" }, children: "source changed" }] : []), + { type: "Anchor", props: { href: routeHref({ mode: "focused", view: `lineage:${f.ref}` }), size: "sm" }, children: "Where from?" }, + ], + }, + ], + })), + }]), + ], + }), + }; +} diff --git a/packages/lifeboard/src/views/types.ts b/packages/lifeboard/src/views/types.ts index 0ad9f2a..de60c4f 100644 --- a/packages/lifeboard/src/views/types.ts +++ b/packages/lifeboard/src/views/types.ts @@ -1,3 +1,4 @@ +import type { KnownPushType } from "@ivanmkc/termchart-core"; import type { Row } from "../log.js"; import type { Store } from "../store.js"; import type { Profile } from "../types.js"; @@ -12,9 +13,10 @@ export interface BoardContext { today: string; } -/** A board, in the same shape a push to the canvas takes. */ +/** A board, in the same shape a push to the canvas takes. Any type the canvas renders is allowed: + * the lineage view is a `flow`, and a pack may bind one to a chart. */ export interface Board { - type: "component" | "panes"; + type: KnownPushType; content: string; title: string; } diff --git a/packages/lifeboard/test/provenance.test.ts b/packages/lifeboard/test/provenance.test.ts new file mode 100644 index 0000000..c12cf1f --- /dev/null +++ b/packages/lifeboard/test/provenance.test.ts @@ -0,0 +1,230 @@ +import "fake-indexeddb/auto"; +import { describe, it, expect } from "vitest"; +import { validateContent } from "@ivanmkc/termchart-core"; +import { openStore } from "../src/store.js"; +import { contentHash, lineageOf, sourceChanged } from "../src/provenance/graph.js"; +import { correct, derivationMutation, flagStale, sourceMutation } from "../src/provenance/record.js"; +import { buildLineageBoard } from "../src/views/lineage.js"; +import { deepLink, linkLabel } from "../src/connectors/deeplinks.js"; +import { add, update, type By } from "../src/types.js"; +import type { Source } from "../src/provenance/types.js"; + +const model: By = { kind: "model", model: "claude-opus-5", confidence: 0.9 }; +const you: By = { kind: "human", profile: "ivan" }; +let n = 0; +const fresh = () => openStore(`lb-prov-${++n}`); + +/** The gate's example: a vaccination date extracted from a PDF page, which was attached to an email. */ +async function vaccinationChain() { + const store = await fresh(); + const email = await sourceMutation({ + space: "shared", connector: "gmail", title: "Re: Sam's booster", + ref: "18f2c9a", excerpt: "…booster given on 14 March…", text: "booster given on 14 March", + }, "src-email"); + const pdf = await sourceMutation({ + space: "shared", connector: "gdrive", title: "Vaccination record.pdf", + ref: "1AbC-drive-id", locator: "2", excerpt: "MMR booster — 14/03/2026", text: "MMR booster 14/03/2026", + }, "src-pdf"); + await store.apply([email.mutation, pdf.mutation], { kind: "code", fn: "intake" }); + + const [fact] = await store.apply( + [add("item", { id: "vacc", space: "shared", list: "health", text: "Sam — MMR booster 14 March 2026", done: false }, [email.ref, pdf.ref])], + model, + ); + await store.apply( + [derivationMutation({ space: "shared", factRef: "item/vacc", entrySeq: fact.seq, step: "extracted", inputs: [email.ref, pdf.ref], by: "model", confidence: 0.82, note: "Read from the certificate." })], + { kind: "code", fn: "intake" }, + ); + return store; +} + +describe("deep links", () => { + it("addresses a page inside a Drive PDF", () => { + expect(deepLink({ connector: "gdrive", id: "1AbC", locator: "2" })).toBe("https://drive.google.com/file/d/1AbC/view#page=2"); + }); + it("opens the document when the connector cannot address inside it", () => { + expect(deepLink({ connector: "gmail", id: "18f2c9a", locator: "3" })).toBe("https://mail.google.com/mail/u/0/#all/18f2c9a"); + expect(deepLink({ connector: "dropbox", id: "/Docs/a b.pdf" })).toContain("preview=%2FDocs%2Fa%20b.pdf"); + }); + it("passes a URL-shaped source through untouched", () => { + expect(deepLink({ connector: "web", id: "https://example.com/a?b=1" })).toBe("https://example.com/a?b=1"); + }); + it("names the place each link goes", () => { + expect(linkLabel("gmail")).toBe("Open in Gmail"); + expect(linkLabel("gdrive")).toBe("Open in Drive"); + }); +}); + +describe("lineageOf", () => { + it("traces a fact back to both documents it came from", async () => { + const store = await vaccinationChain(); + const { nodes, edges } = lineageOf(store, "item/vacc"); + const sources = nodes.filter((x) => x.kind === "source"); + expect(sources.map((s) => s.label).sort()).toEqual(["Re: Sam's booster", "Vaccination record.pdf"]); + expect(sources.find((s) => s.label.includes("pdf"))!.url).toContain("#page=2"); + expect(edges.length).toBeGreaterThanOrEqual(3); + }); + + it("says which step it was, who did it and how sure they were", async () => { + const store = await vaccinationChain(); + const step = lineageOf(store, "item/vacc").nodes.find((x) => x.kind === "step")!; + expect(step.label).toBe("extracted"); + expect(step.detail).toContain("claude-opus-5"); + expect(step.detail).toContain("82% confident"); + }); + + it("shows a correction and the value it replaced, not just the latest", async () => { + const store = await vaccinationChain(); + await correct(store, "item/vacc", { text: "Sam — MMR booster 14 April 2026" }, you); + const { nodes } = lineageOf(store, "item/vacc"); + const steps = nodes.filter((x) => x.kind === "step"); + expect(steps.map((s) => s.label)).toEqual(["extracted", "corrected"]); + expect(steps[0].superseded).toBe(true); + expect(steps[1].detail).toContain("by ivan"); + expect(nodes.find((x) => x.kind === "fact")!.label).toContain("14 April"); + }); + + it("traces a shopping line back to the recipe that put it there", async () => { + const store = await fresh(); + await store.apply([add("recipe", { id: "r1", space: "shared", title: "Pancakes", servings: 6, ingredients: [], steps: [] })], model); + await store.apply([add("item", { id: "flour", space: "shared", text: "300 g plain flour", done: false }, ["recipe/r1"])], { kind: "code", fn: "mergeIntoList" }); + const { nodes } = lineageOf(store, "item/flour"); + expect(nodes.map((x) => x.label)).toContain("Pancakes"); + expect(nodes.find((x) => x.kind === "step")!.detail).toContain("mergeIntoList"); + }); + + it("does not loop forever on a fact that refers to itself", async () => { + const store = await fresh(); + await store.apply([add("item", { id: "a", space: "shared", text: "A", done: false }, ["item/a"])], you); + expect(() => lineageOf(store, "item/a")).not.toThrow(); + }); + + it("returns nothing traceable for a fact that never existed", async () => { + expect(lineageOf(await fresh(), "item/ghost").nodes).toEqual([]); + }); +}); + +describe("the lineage board", () => { + it("is a flow board the shared validator accepts", async () => { + const store = await vaccinationChain(); + const board = buildLineageBoard(store, "item/vacc"); + expect(board.type).toBe("flow"); + expect(validateContent("flow", board.content)).toBeNull(); + }); + + it("carries the link to tap on the source node", async () => { + const store = await vaccinationChain(); + const spec = JSON.parse(buildLineageBoard(store, "item/vacc").content); + const pdf = spec.nodes.find((x: { label: string }) => x.label.includes("pdf")); + expect(pdf.url).toContain("drive.google.com"); + expect(spec.legend).toEqual(expect.arrayContaining([{ color: expect.any(String), label: "Document" }])); + }); + + it("says so plainly when there is nothing to trace", async () => { + const board = buildLineageBoard(await fresh(), "item/ghost"); + expect(board.type).toBe("component"); + expect(board.content).toContain("Nothing to trace"); + expect(validateContent("component", board.content)).toBeNull(); + }); +}); + +describe("a source that changed", () => { + it("notices, by comparing hashes", async () => { + const store = await vaccinationChain(); + const pdf = store.get("source", "src-pdf")!; + expect(await sourceChanged(pdf, "MMR booster 14/03/2026")).toBe(false); + expect(await sourceChanged(pdf, "MMR booster 14/04/2026")).toBe(true); + }); + + it("says no when there was never a hash to compare", async () => { + const store = await fresh(); + const { mutation } = await sourceMutation({ space: "shared", connector: "web", title: "A page", ref: "https://x" }, "s1"); + await store.apply([mutation], you); + expect(await sourceChanged(store.get("source", "s1")!, "anything at all")).toBe(false); + }); + + it("flags the facts drawn from it, and does not re-read them", async () => { + const store = await vaccinationChain(); + const before = store.get<{ text: string }>("item", "vacc")!.text; + const touched = await flagStale(store, "source/src-pdf", { kind: "code", fn: "watch" }); + + expect(touched).toEqual(["item/vacc"]); + expect(store.get<{ needsReview?: boolean; text: string }>("item", "vacc")).toMatchObject({ needsReview: true, text: before }); + expect(store.get("source", "src-pdf")).toMatchObject({ stale: true }); + expect(lineageOf(store, "item/vacc").nodes.find((x) => x.kind === "source" && x.label.includes("pdf"))!.stale).toBe(true); + }); + + it("hashes the same text to the same value", async () => { + expect(await contentHash("abc")).toBe(await contentHash("abc")); + expect(await contentHash("abc")).not.toBe(await contentHash("abd")); + }); +}); + +describe("correct", () => { + it("keeps both the correction and the original", async () => { + const store = await vaccinationChain(); + await correct(store, "item/vacc", { text: "Sam — MMR booster 14 April 2026" }, you); + const history = store.history("item/vacc"); + expect(history.map((e) => e.op)).toEqual(["assert", "correct"]); + expect((history[0] as { value: { text: string } }).value.text).toContain("14 March"); + }); + + it("records that a human made the correction, so nothing re-derives over it", async () => { + const store = await vaccinationChain(); + await correct(store, "item/vacc", { text: "x" }, you); + const derivations = store.list<{ step: string; by: string }>("derivation", { match: { factRef: "item/vacc" } }); + expect(derivations.map((d) => [d.step, d.by])).toEqual([["extracted", "model"], ["corrected", "human"]]); + }); + + it("does nothing to a fact that is not there", async () => { + expect(await correct(await fresh(), "item/ghost", { text: "x" }, you)).toBe(false); + }); +}); + +describe("finding your way to a lineage", () => { + it("lists traced facts newest first, with a link to each chain", async () => { + const store = await vaccinationChain(); + const { buildSourcesBoard, tracedFacts } = await import("../src/views/sources.js"); + const { ensureProfiles, spacesFor } = await import("../src/profiles.js"); + const profile = await ensureProfiles(store); + const ctx = { store, profile, spaces: [...spacesFor(profile.id), "shared"], today: "2026-09-02" }; + + expect(tracedFacts(store, ctx.spaces).map((f) => f.ref)).toEqual(["item/vacc"]); + const board = buildSourcesBoard(ctx); + expect(validateContent("component", board.content)).toBeNull(); + expect(board.content).toContain("#lb:focused:lineage:item/vacc"); + expect(board.content).toContain("read by claude-opus-5"); + }); + + it("marks the ones whose document has changed, and says nothing was re-read", async () => { + const store = await vaccinationChain(); + await flagStale(store, "source/src-pdf", { kind: "code", fn: "watch" }); + const { buildSourcesBoard } = await import("../src/views/sources.js"); + const { ensureProfiles, spacesFor } = await import("../src/profiles.js"); + const profile = await ensureProfiles(store); + const board = buildSourcesBoard({ store, profile, spaces: spacesFor(profile.id), today: "2026-09-02" }); + expect(board.content).toContain("source changed"); + expect(board.content).toMatch(/nothing was re-read automatically/); + }); + + it("says so plainly when nothing has been traced yet", async () => { + const store = await fresh(); + const { buildSourcesBoard } = await import("../src/views/sources.js"); + const { ensureProfiles, spacesFor } = await import("../src/profiles.js"); + const profile = await ensureProfiles(store); + const board = buildSourcesBoard({ store, profile, spaces: spacesFor(profile.id), today: "2026-09-02" }); + expect(board.content).toMatch(/nothing here was taken from a document yet/i); + expect(validateContent("component", board.content)).toBeNull(); + }); +}); + +describe("in-board links", () => { + it("recognises a route link and leaves an ordinary link alone", async () => { + const { routeFromHref, routeHref } = await import("../src/views/links.js"); + expect(routeFromHref(routeHref({ mode: "focused", view: "lineage:item/vacc" }))) + .toEqual({ mode: "focused", view: "lineage:item/vacc" }); + expect(routeFromHref("https://drive.google.com/file/d/1AbC/view")).toBeNull(); + expect(routeFromHref(null)).toBeNull(); + expect(routeFromHref("#lb:nonsense")).toBeNull(); + }); +}); diff --git a/packages/lifeboard/test/shell.test.ts b/packages/lifeboard/test/shell.test.ts index 6e220d0..b077df0 100644 --- a/packages/lifeboard/test/shell.test.ts +++ b/packages/lifeboard/test/shell.test.ts @@ -35,7 +35,7 @@ describe("shell", () => { const shell = await startShell({ regions: r, store, prefs, now: () => new Date(2026, 8, 2) }); expect(shell.route()).toEqual({ mode: "grid", view: "home" }); expect(shell.board().type).toBe("panes"); - expect(r.rail.querySelectorAll(".lb-rail-item")).toHaveLength(5); + expect(r.rail.querySelectorAll(".lb-rail-item")).toHaveLength(6); expect(r.rail.querySelector(".lb-rail-item.active")?.textContent).toBe("Home"); expect(r.console.querySelector("form")).not.toBeNull(); shell.destroy(); @@ -174,3 +174,31 @@ describe("transcript persistence", () => { shell.destroy(); }); }); + +describe("tapping a link inside a board", () => { + it("navigates instead of leaving the app", async () => { + const { store, prefs, r } = await boot(); + const shell = await startShell({ regions: r, store, prefs, now: () => new Date(2026, 8, 2) }); + const a = document.createElement("a"); + a.setAttribute("href", "#lb:focused:groceries"); + r.board.append(a); + a.dispatchEvent(new Event("click", { bubbles: true, cancelable: true })); + await new Promise((res) => setTimeout(res, 10)); + expect(shell.route()).toEqual({ mode: "focused", view: "groceries" }); + shell.destroy(); + }); + + it("leaves an ordinary link to the browser", async () => { + const { store, prefs, r } = await boot(); + const shell = await startShell({ regions: r, store, prefs, now: () => new Date(2026, 8, 2) }); + const before = shell.route(); + const a = document.createElement("a"); + a.setAttribute("href", "https://drive.google.com/file/d/1AbC/view"); + r.board.append(a); + const ev = new Event("click", { bubbles: true, cancelable: true }); + a.dispatchEvent(ev); + expect(ev.defaultPrevented).toBe(false); + expect(shell.route()).toEqual(before); + shell.destroy(); + }); +});