feat(lifeboard): provenance — every fact can say where it came from

packages/lifeboard/src/connectors/deeplinks.ts

@@ -0,0 +1,62 @@
+// Deep links, as data. Every connector addresses a document differently and the formats change
+// without warning, so they live in a table rather than in code paths — a broken link is then a
+// one-line fix, not a hunt through three modules.
+//
+// The locator is the part inside the document: a page in a PDF, a paragraph in an email. Where a
+// connector supports addressing it, the template uses it; where it does not, the link opens the
+// document and the excerpt on the lineage card tells you what to look for.
+
+export type ConnectorId = "gmail" | "gdrive" | "dropbox" | "photo" | "web" | "calendar" | "manual";
+
+export interface LinkTemplate {
+  /** {id} is the document, {locator} the place inside it, {query} a search fallback. */
+  document: string;
+  withLocator?: string;
+  label: string;
+}
+
+export const LINKS: Record<ConnectorId, LinkTemplate> = {
+  // A message id addresses the thread; rfc822msgid search is the fallback when only the header id
+  // is known, which is what an IMAP-shaped extraction usually has.
+  gmail: {
+    document: "https://mail.google.com/mail/u/0/#all/{id}",
+    label: "Open in Gmail",
+  },
+  gdrive: {
+    document: "https://drive.google.com/file/d/{id}/view",
+    withLocator: "https://drive.google.com/file/d/{id}/view#page={locator}",
+    label: "Open in Drive",
+  },
+  dropbox: {
+    document: "https://www.dropbox.com/home?preview={id}",
+    label: "Open in Dropbox",
+  },
+  photo: { document: "{id}", label: "Open the photo" },
+  web: { document: "{id}", label: "Open the page" },
+  calendar: {
+    document: "https://calendar.google.com/calendar/u/0/r/eventedit/{id}",
+    label: "Open in Calendar",
+  },
+  manual: { document: "", label: "Entered by hand" },
+};
+
+export interface LinkTarget {
+  connector: ConnectorId;
+  id: string;
+  locator?: string;
+}
+
+/** The URL to open, or "" when the source is not something that can be opened. */
+export function deepLink(target: LinkTarget): string {
+  const t = LINKS[target.connector];
+  if (!t) return "";
+  const template = target.locator && t.withLocator ? t.withLocator : t.document;
+  return template.replace("{id}", encodeId(target.connector, target.id)).replace("{locator}", encodeURIComponent(target.locator ?? ""));
+}
+
+export const linkLabel = (connector: ConnectorId): string => LINKS[connector]?.label ?? "Open";
+
+/** Photos and web pages are already URLs; ids inside a provider's namespace need escaping. */
+function encodeId(connector: ConnectorId, id: string): string {
+  return connector === "photo" || connector === "web" ? id : encodeURIComponent(id);
+}

packages/lifeboard/src/provenance/graph.ts

@@ -0,0 +1,121 @@
+// "Where did this come from?" answered from the log rather than from a side table.
+//
+// The log already records, for every change: what it was derived from (`from`), who made it (`by`),
+// and what it replaced. That is most of a derivation graph, recorded at the moment the fact was
+// created because nothing else was possible — which is the only way provenance ever works. A
+// `derivation` record adds the two things the log cannot infer: which KIND of step it was, and how
+// confident a model was.
+//
+// One rule runs through this file: a source that has changed is FLAGGED, never silently re-read.
+// Re-deriving from a changed document behind someone's back is how a vaccination date quietly
+// becomes a different date.
+
+import type { Store } from "../store.js";
+import { deepLink, linkLabel } from "../connectors/deeplinks.js";
+import type { Entry } from "../types.js";
+import { parseRef } from "../types.js";
+import type { Derivation, Lineage, LineageEdge, LineageNode, Source, Step } from "./types.js";
+
+const MAX_DEPTH = 6;
+
+export function lineageOf(store: Store, factRef: string): Lineage {
+  const nodes = new Map<string, LineageNode>();
+  const edges: LineageEdge[] = [];
+  walk(store, factRef, nodes, edges, 0, new Set());
+  return { nodes: [...nodes.values()], edges };
+}
+
+function walk(store: Store, ref: string, nodes: Map<string, LineageNode>, edges: LineageEdge[], depth: number, seen: Set<string>): void {
+  if (depth > MAX_DEPTH || seen.has(ref)) return;
+  seen.add(ref);
+
+  const { collection, id } = parseRef(ref);
+  if (collection === "source") {
+    addSource(store, id, nodes);
+    return;
+  }
+
+  const history = store.history(ref);
+  if (history.length === 0) return;
+
+  const derivations = store.list<Derivation>("derivation", { match: { factRef: ref } });
+  const current = history[history.length - 1];
+  nodes.set(ref, {
+    id: ref,
+    kind: "fact",
+    label: describeValue(current),
+    detail: `${collection} · ${current.op === "retract" ? "removed" : "as it stands now"}`,
+  });
+
+  // Every entry is a step. Corrections keep the value they replaced visible: the point of a lineage
+  // view is to show that a number changed, not to present the latest one as if it always was.
+  for (const entry of history) {
+    const stepId = `${ref}#${entry.seq}`;
+    const derivation = derivations.find((d) => d.entrySeq === entry.seq);
+    nodes.set(stepId, {
+      id: stepId,
+      kind: "step",
+      label: stepLabel(entry, derivation?.step),
+      detail: [author(entry), derivation?.confidence !== undefined ? `${Math.round(derivation.confidence * 100)}% confident` : "", derivation?.note ?? ""].filter(Boolean).join(" · "),
+      ...(entry.op === "correct" || entry !== current ? { superseded: entry !== current } : {}),
+    });
+    edges.push({ id: `${stepId}->${ref}`, source: stepId, target: ref });
+
+    for (const input of (entry.op === "retract" ? [] : entry.from) ?? []) {
+      edges.push({ id: `${input}->${stepId}`, source: input, target: stepId, label: "from" });
+      walk(store, input, nodes, edges, depth + 1, seen);
+    }
+  }
+}
+
+function addSource(store: Store, id: string, nodes: Map<string, LineageNode>): void {
+  const source = store.get<Source>("source", id);
+  const ref = `source/${id}`;
+  if (!source) {
+    nodes.set(ref, { id: ref, kind: "source", label: "A source that is no longer here" });
+    return;
+  }
+  nodes.set(ref, {
+    id: ref,
+    kind: "source",
+    label: source.title,
+    detail: [source.excerpt, source.locator ? `page ${source.locator}` : "", linkLabel(source.connector)].filter(Boolean).join(" · "),
+    url: deepLink({ connector: source.connector, id: source.ref, locator: source.locator }),
+    ...(source.stale ? { stale: true } : {}),
+  });
+}
+
+/** SHA-256 of the document's text. Used only to notice a change; nothing re-reads on its own. */
+export async function contentHash(text: string): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+  return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+/**
+ * Has the document behind this source changed? Answering yes marks the facts drawn from it as
+ * needing a look — it never re-extracts them. A model-derived fact especially: re-running the
+ * extraction could change a date nobody asked to change.
+ */
+export async function sourceChanged(source: Source, currentText: string): Promise<boolean> {
+  if (!source.contentHash) return false;
+  return (await contentHash(currentText)) !== source.contentHash;
+}
+
+const author = (e: Entry): string =>
+  e.by.kind === "human" ? `by ${e.by.profile}` : e.by.kind === "code" ? `computed by ${e.by.fn}` : `from ${e.by.model}`;
+
+function stepLabel(entry: Entry, step?: Step): string {
+  if (step) return step;
+  if (entry.op === "assert") return entry.by.kind === "human" ? "entered" : entry.by.kind === "code" ? "computed" : "extracted";
+  if (entry.op === "correct") return "corrected";
+  return "removed";
+}
+
+function describeValue(entry: Entry): string {
+  if (entry.op === "retract") return "(removed)";
+  const v = entry.value as Record<string, unknown>;
+  for (const key of ["text", "title", "name", "summary"]) {
+    if (typeof v[key] === "string" && v[key]) return v[key] as string;
+  }
+  return parseRef(entry.ref).id;
+}

packages/lifeboard/src/provenance/record.ts

@@ -0,0 +1,115 @@
+// The producer side of provenance: helpers every path that creates a fact uses, so the graph is
+// recorded at the moment the fact is created. Threading this through the producers is the whole
+// job — a derivation bolted on afterwards is a guess, and a guess about where a vaccination date
+// came from is worse than nothing.
+
+import type { Store } from "../store.js";
+import { add, update, type By, type CollectionName, type Mutation } from "../types.js";
+import { contentHash } from "./graph.js";
+import type { Derivation, Source, Step } from "./types.js";
+import type { ConnectorId } from "../connectors/deeplinks.js";
+
+export interface RecordSourceInput {
+  space: string;
+  connector: ConnectorId;
+  title: string;
+  ref: string;
+  locator?: string;
+  excerpt?: string;
+  /** The document's text, hashed so a later change can be noticed. */
+  text?: string;
+  retrievedAt?: string;
+}
+
+/** Register a document. Returns the mutation and the ref facts should point `from` at. */
+export async function sourceMutation(input: RecordSourceInput, id: string, now = new Date()): Promise<{ mutation: Mutation; ref: string }> {
+  const value: Omit<Source, "createdAt" | "updatedAt"> = {
+    id,
+    space: input.space,
+    connector: input.connector,
+    title: input.title,
+    ref: input.ref,
+    retrievedAt: input.retrievedAt ?? now.toISOString(),
+    ...(input.locator ? { locator: input.locator } : {}),
+    ...(input.excerpt ? { excerpt: input.excerpt } : {}),
+    ...(input.text ? { contentHash: await contentHash(input.text) } : {}),
+  };
+  return { mutation: add("source", value as unknown as Parameters<typeof add>[1]), ref: `source/${id}` };
+}
+
+export interface DerivationInput {
+  space: string;
+  factRef: string;
+  /** The log entry this explains. */
+  entrySeq: number;
+  step: Step;
+  inputs: string[];
+  by: Derivation["by"];
+  confidence?: number;
+  note?: string;
+  at?: string;
+}
+
+export const derivationMutation = (input: DerivationInput, now = new Date()): Mutation =>
+  add("derivation", {
+    space: input.space,
+    factRef: input.factRef,
+    entrySeq: input.entrySeq,
+    step: input.step,
+    inputs: input.inputs,
+    by: input.by,
+    at: input.at ?? now.toISOString(),
+    ...(input.confidence !== undefined ? { confidence: input.confidence } : {}),
+    ...(input.note ? { note: input.note } : {}),
+  });
+
+/**
+ * Change a fact by hand. The correction and the value it replaced both stay in the log, and the
+ * correction is marked as human so nothing later re-derives over the top of it.
+ */
+export async function correct(
+  store: Store,
+  factRef: string,
+  patch: Record<string, unknown>,
+  who: By,
+  space = "shared",
+): Promise<boolean> {
+  const [collection, ...rest] = factRef.split("/");
+  const id = rest.join("/");
+  if (!store.get(collection as CollectionName, id)) return false;
+  const entries = await store.apply([update(collection as CollectionName, id, patch)], who);
+  if (entries.length === 0) return false;
+  await store.apply(
+    [derivationMutation({
+      space,
+      factRef,
+      entrySeq: entries[0].seq,
+      step: "corrected",
+      inputs: [],
+      by: who.kind === "human" ? "human" : who.kind === "code" ? "code" : "model",
+      note: "Corrected by hand.",
+    })],
+    who,
+  );
+  return true;
+}
+
+/**
+ * Mark every fact drawn from a source as needing a look. Flagging, never re-reading: re-running an
+ * extraction against a changed document could alter a value nobody asked to alter.
+ */
+export async function flagStale(store: Store, sourceRef: string, by: By): Promise<string[]> {
+  const touched = new Set<string>();
+  for (const entry of store.history()) {
+    if (entry.op === "retract" || !entry.from?.includes(sourceRef)) continue;
+    touched.add(entry.ref);
+  }
+  const { id } = { id: sourceRef.slice(sourceRef.indexOf("/") + 1) };
+  const mutations: Mutation[] = [update("source", id, { stale: true })];
+  for (const ref of touched) {
+    const [collection, ...rest] = ref.split("/");
+    mutations.push(update(collection as CollectionName, rest.join("/"), { needsReview: true }));
+  }
+  await store.apply(mutations, by);
+  return [...touched];
+}

packages/lifeboard/src/provenance/types.ts

@@ -0,0 +1,61 @@
+import type { Base } from "../types.js";
+import type { ConnectorId } from "../connectors/deeplinks.js";
+
+/** A document a fact came from. Recorded when the fact is created, never reconstructed later. */
+export interface Source extends Base {
+  connector: ConnectorId;
+  title: string;
+  /** The provider's id for the document — the deep link is built from it, not stored pre-built,
+   *  so a change in a provider's URL format fixes every old source at once. */
+  ref: string;
+  /** Where inside the document: a page number, a paragraph index. */
+  locator?: string;
+  excerpt?: string;
+  retrievedAt: string;
+  /** For noticing that the document changed. Compared, never used to re-read it automatically. */
+  contentHash?: string;
+  /** Set once the document is known to have changed. The facts drawn from it are flagged too. */
+  stale?: boolean;
+}
+
+export type Step = "extracted" | "parsed" | "scaled" | "merged" | "computed" | "entered" | "corrected";
+
+/** How one fact came from others. The log's own `from` and `by` carry most of this; a derivation
+ *  record adds the step and the confidence, which the log cannot infer. */
+export interface Derivation extends Base {
+  factRef: string;
+  /** The sequence number of the log entry this explains. Identity, not a timestamp: two things can
+   *  happen in the same millisecond, and pairing a derivation to the wrong entry mislabels the
+   *  history of a fact someone is trying to check. */
+  entrySeq: number;
+  step: Step;
+  inputs: string[];
+  by: "model" | "code" | "human";
+  confidence?: number;
+  note?: string;
+  at: string;
+}
+
+export interface LineageNode {
+  id: string;
+  kind: "fact" | "step" | "source";
+  label: string;
+  detail?: string;
+  /** Present on source nodes: where tapping takes you. */
+  url?: string;
+  /** True when the underlying document has changed since the fact was taken from it. */
+  stale?: boolean;
+  superseded?: boolean;
+}
+
+export interface LineageEdge {
+  id: string;
+  source: string;
+  target: string;
+  label?: string;
+}
+
+export interface Lineage {
+  nodes: LineageNode[];
+  edges: LineageEdge[];
+}

packages/lifeboard/src/rail.ts

@@ -15,6 +15,7 @@ export const DEFAULT_RAIL: RailItem[] = [
   { label: "Today", route: { mode: "focused", view: "today" } },
   { label: "Meals", route: { mode: "focused", view: "meals" } },
   { label: "Shopping", route: { mode: "focused", view: "groceries" } },
+  { label: "Sources", route: { mode: "focused", view: "sources" } },
   { label: "Connect", route: { mode: "focused", view: "connections" } },
 ];