Literature plane slice 1: the paper record + the unified corpus fold (#405)

packages/amico-run/src/papers.ts

@@ -0,0 +1,205 @@
+// papers.ts — the unified literature corpus fold (#405): one read-only view
+// over vault papers/ notes + the library PDF store. Collected, unified,
+// deduped by identity (REPORTED, never merged — merging is a human promote
+// act), content-addressed join (sha256 on read; filenames stay human),
+// orphans surfaced both directions. The fold never writes.
+//
+// Zero-dep note: vault note frontmatter is a YAML subset (flat scalars,
+// quoted strings, inline lists, null) — a ~50-line reader beats a dependency
+// (the runstatus.ts TOML-subset precedent, invariant 7). If a note needs
+// richer YAML, widen the subset with a test, not a library.
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { validate, studioPathsOrLegacy } from "@amicode/schema";
+import type { StudioPaths } from "@amicode/schema";
+
+/** Parse a note's --- frontmatter fence (flat YAML subset). Throws on a
+ *  missing fence; unknown value shapes land as strings. */
+export function parseFrontmatter(text: string): Record<string, unknown> {
+  const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!m) throw new Error("missing --- frontmatter block");
+  const out: Record<string, unknown> = {};
+  for (const line of m[1]!.split(/\r?\n/)) {
+    if (!line.trim() || line.trim().startsWith("#")) continue;
+    const kv = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
+    if (!kv) throw new Error(`unparseable frontmatter line: ${line}`);
+    const [, key, raw] = kv;
+    out[key] = parseValue(raw!.trim());
+  }
+  return out;
+}
+
+function parseValue(raw: string): unknown {
+  if (raw === "" || raw === "null" || raw === "~") return null;
+  if (raw.startsWith("[") && raw.endsWith("]")) {
+    return raw
+      .slice(1, -1)
+      .split(",")
+      .map((s) => s.trim())
+      .filter((s) => s !== "")
+      .map((s) => (s.startsWith('"') && s.endsWith('"') ? s.slice(1, -1) : s.startsWith("'") && s.endsWith("'") ? s.slice(1, -1) : s));
+  }
+  if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1);
+  return raw;
+}
+
+export interface PaperRecord {
+  file: string;
+  title: string;
+  authors: string[];
+  arxiv?: string;
+  doi?: string;
+  status: "staged" | "distilled"; // absent frontmatter = distilled (historical)
+  tags: string[];
+  systems: string[];
+  relevance?: string;
+  frontmatter: Record<string, unknown>;
+  pdf?: { file: string; sha256: string };
+}
+
+export interface CorpusReport {
+  papers: PaperRecord[];
+  /** same identity seen in multiple notes — reported, never merged */
+  duplicates: { key: string; files: string[] }[];
+  /** notes whose frontmatter fails the library-paper contract */
+  invalid: { file: string; errors: string[] }[];
+  /** library PDFs no record claims */
+  orphanPdfs: { file: string; sha256: string }[];
+  /** records with no PDF in the library (the acquisition to-do list) */
+  recordsWithoutPdf: { file: string; title: string; arxiv?: string; doi?: string }[];
+}
+
+const sha256 = (buf: Buffer) => createHash("sha256").update(buf).digest("hex");
+
+/** arxiv "1711.09641v2" → "1711.09641" — version suffixes normalize at the
+ *  fold (the schema stays strict on the canonical form). */
+function normalizeArxiv(id: string): string {
+  return id.replace(/v\d+$/, "");
+}
+
+/** Fold the corpus: every <vaults-root>/<mount>/papers/*.md note, validated,
+ *  unified, joined against the library's PDFs. Read-only; absence degrades
+ *  to empty everywhere. */
+export function foldCorpus(vaultRoots: string[], libraryRoot: string): CorpusReport {
+  const report: CorpusReport = { papers: [], duplicates: [], invalid: [], orphanPdfs: [], recordsWithoutPdf: [] };
+
+  // 1. collect + validate notes across every mount of every root
+  const byIdentity = new Map<string, PaperRecord[]>();
+  for (const root of vaultRoots) {
+    if (!existsSync(root)) continue;
+    let mounts: string[] = [];
+    try {
+      mounts = readdirSync(root, { withFileTypes: true })
+        .filter((d) => d.isDirectory())
+        .map((d) => join(root, d.name));
+    } catch {
+      continue;
+    }
+    for (const mount of mounts) {
+      const papersDir = join(mount, "papers");
+      if (!existsSync(papersDir)) continue;
+      let files: string[] = [];
+      try {
+        files = readdirSync(papersDir).filter((f) => f.endsWith(".md"));
+      } catch {
+        continue;
+      }
+      for (const f of files) {
+        if (f.startsWith("MERGED-INTO-")) continue; // merge tombstones are retired records
+        const file = join(papersDir, f);
+        let fm: Record<string, unknown>;
+        try {
+          fm = parseFrontmatter(readFileSync(file, "utf8"));
+        } catch (e) {
+          report.invalid.push({ file, errors: [String(e)] });
+          continue;
+        }
+        const v = validate(fm, "library-paper");
+        if (!v.ok) {
+          report.invalid.push({ file, errors: v.errors });
+          continue;
+        }
+        const rec: PaperRecord = {
+          file,
+          title: fm.title as string,
+          authors: fm.authors as string[],
+          arxiv: fm.arxiv ? normalizeArxiv(fm.arxiv as string) : undefined,
+          doi: fm.doi as string | undefined,
+          status: (fm.status as "staged" | "distilled") ?? "distilled",
+          tags: (fm.tags as string[]) ?? [],
+          systems: (fm.systems as string[]) ?? [],
+          relevance: fm.relevance as string | undefined,
+          frontmatter: fm,
+        };
+        report.papers.push(rec);
+        const key = rec.arxiv ? `arxiv:${rec.arxiv}` : rec.doi ? `doi:${rec.doi}` : null;
+        if (key) {
+          const bucket = byIdentity.get(key) ?? [];
+          bucket.push(rec);
+          byIdentity.set(key, bucket);
+        }
+      }
+    }
+  }
+  for (const [key, bucket] of byIdentity)
+    if (bucket.length > 1) report.duplicates.push({ key, files: bucket.map((b) => b.file) });
+
+  // 2. library PDFs: content-addressed, identity-joined by filename
+  const pdfs: { file: string; sha256: string }[] = [];
+  if (existsSync(libraryRoot)) {
+    try {
+      for (const f of readdirSync(libraryRoot)) {
+        const file = join(libraryRoot, f);
+        try {
+          if (!statSync(file).isFile()) continue;
+          pdfs.push({ file, sha256: sha256(readFileSync(file)) });
+        } catch {
+          /* unreadable file — skip */
+        }
+      }
+    } catch {
+      /* unreadable root — no pdfs */
+    }
+  }
+  const claimed = new Set<string>();
+  for (const p of report.papers) {
+    const match = p.arxiv
+      ? pdfs.find((x) => basenameContainsIdentity(x.file, p.arxiv!))
+      : p.doi
+        ? pdfs.find((x) => x.file.includes(sanitizeDoi(p.doi!)))
+        : undefined;
+    if (match) {
+      p.pdf = { file: match.file, sha256: match.sha256 };
+      claimed.add(match.file);
+    } else {
+      report.recordsWithoutPdf.push({ file: p.file, title: p.title, arxiv: p.arxiv, doi: p.doi });
+    }
+  }
+  report.orphanPdfs = pdfs.filter((x) => !claimed.has(x.file));
+  return report;
+}
+
+function basenameContainsIdentity(file: string, arxivId: string): boolean {
+  const base = file.replace(/^.*[\\/]/, "");
+  // boundary-safe: "1711.09641.pdf", "1711.09641v2.pdf", "arXiv-1711.09641(1).pdf"
+  const re = new RegExp(`(^|[^0-9])${escapeRe(arxivId)}(v\\d+)?([^0-9]|$)`);
+  return re.test(base);
+}
+
+function sanitizeDoi(doi: string): string {
+  return escapeRe(doi.replace(/^https?:\/\/(dx\.)?doi\.org\//, ""));
+}
+
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** The corpus over THIS machine's studio ladder (manifest → legacy). The
+ *  library root stays the legacy ~/.amico/library until the manifest grows a
+ *  library field (v2 — the installation spec keeps PDFs as library state). */
+export function foldStudioCorpus(paths?: StudioPaths): CorpusReport {
+  const p = paths ?? studioPathsOrLegacy();
+  return foldCorpus([p.vaultsRoot], join(homedir(), ".amico", "library"));
+}

packages/amico-run/src/papers_list.ts

@@ -0,0 +1,74 @@
+// papers_list.ts — the `amico papers list` body: fold + filter + render.
+// Pure rendering decisions live in papers_render.ts; the fold is papers.ts.
+import { papersFilters, renderCorpusTable } from "./papers_render.js";
+import { foldCorpus } from "./papers.js";
+import { studioPathsOrLegacy } from "@amicode/schema";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { VerbResult } from "./verbs.js";
+import type { CorpusReport, PaperRecord } from "./papers.js";
+
+function flagValue(argv: string[], name: string): string | undefined {
+  const i = argv.indexOf(name);
+  return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined;
+}
+
+function corpusCounts(corpus: CorpusReport, shown: number): Record<string, number> {
+  return {
+    papers: shown,
+    total: corpus.papers.length,
+    duplicates: corpus.duplicates.length,
+    invalid: corpus.invalid.length,
+    orphan_pdfs: corpus.orphanPdfs.length,
+    records_without_pdf: corpus.recordsWithoutPdf.length,
+  };
+}
+
+export function papersList(argv: string[]): VerbResult {
+  const asJson = argv.includes("--json");
+  const filters = papersFilters({
+    status: flagValue(argv, "--status"),
+    tag: flagValue(argv, "--tag"),
+    platform: flagValue(argv, "--platform"),
+    q: flagValue(argv, "--q"),
+  });
+
+  // Hermetic escapes win; production roots ride the studio ladder.
+  const vaults = process.env.AMICO_PAPERS_VAULTS ?? studioPathsOrLegacy().vaultsRoot;
+  const library = process.env.AMICO_PAPERS_LIBRARY ?? join(homedir(), ".amico", "library");
+  const corpus = foldCorpus([vaults], library);
+  const papers: PaperRecord[] = corpus.papers.filter(filters);
+
+  const payload = papers.map((p) => ({
+    title: p.title,
+    authors: p.authors,
+    arxiv: p.arxiv ?? null,
+    doi: p.doi ?? null,
+    status: p.status,
+    relevance: p.relevance ?? null,
+    systems: p.systems,
+    tags: p.tags,
+    file: p.file,
+    pdf: p.pdf ? { file: p.pdf.file, sha256: p.pdf.sha256 } : null,
+  }));
+
+  if (asJson) {
+    return {
+      json: {
+        ok: true,
+        papers: payload,
+        counts: corpusCounts(corpus, papers.length),
+        duplicates: corpus.duplicates,
+        invalid: corpus.invalid,
+        orphan_pdfs: corpus.orphanPdfs,
+        records_without_pdf: corpus.recordsWithoutPdf,
+      },
+      code: 0,
+    };
+  }
+
+  return {
+    json: { ok: true, table: renderCorpusTable(papers, corpus), counts: corpusCounts(corpus, papers.length) },
+    code: 0,
+  };
+}

packages/amico-run/src/papers_render.ts

@@ -0,0 +1,44 @@
+// papers_render.ts — pure filter + table rendering for `amico papers list`.
+// No I/O; trivially testable.
+import type { CorpusReport, PaperRecord } from "./papers.js";
+
+export interface FilterSpec {
+  status?: string;
+  tag?: string;
+  platform?: string;
+  q?: string;
+}
+
+/** Build the predicate chain — each present flag ANDs. */
+export function papersFilters(spec: FilterSpec): (p: PaperRecord) => boolean {
+  const tests: ((p: PaperRecord) => boolean)[] = [];
+  if (spec.status) {
+    const want = spec.status as PaperRecord["status"];
+    tests.push((p) => p.status === want);
+  }
+  if (spec.tag) tests.push((p) => p.tags.includes(spec.tag!));
+  if (spec.platform) tests.push((p) => p.systems.includes(spec.platform!));
+  if (spec.q) {
+    const needle = spec.q.toLowerCase();
+    tests.push((p) =>
+      [p.title, ...p.authors, p.arxiv ?? "", p.doi ?? "", ...p.tags].some((s) => s.toLowerCase().includes(needle)),
+    );
+  }
+  return (p) => tests.every((t) => t(p));
+}
+
+/** The human table: title · identity · status · systems · pdf?. */
+export function renderCorpusTable(papers: PaperRecord[], corpus: CorpusReport): string {
+  const ident = (p: PaperRecord) => p.arxiv ? `arXiv:${p.arxiv}` : p.doi ? `doi:${p.doi}` : "?";
+  const rows = papers.map((p) => [p.title.slice(0, 52), ident(p), p.status, p.systems.join(","), p.pdf ? "pdf" : "—"]);
+  const width = [56, 22, 8, 16, 3].map((w, i) => Math.max(w, ...rows.map((r) => r[i]!.length)));
+  const lines = [
+    `${"title".padEnd(width[0]!)}  ${"identity".padEnd(width[1]!)}  ${"status".padEnd(width[2]!)}  ${"systems".padEnd(width[3]!)}  pdf`,
+    ...rows.map((r) => r.map((c, i) => c.padEnd(width[i]!)).join("  ")),
+  ];
+  if (corpus.duplicates.length) lines.push(``, `duplicates: ${corpus.duplicates.map((d) => `${d.key} ×${d.files.length}`).join(", ")}`);
+  if (corpus.invalid.length) lines.push(`invalid notes: ${corpus.invalid.length} (amico papers list --json for files)`);
+  if (corpus.orphanPdfs.length) lines.push(`orphan pdfs: ${corpus.orphanPdfs.length}`);
+  if (corpus.recordsWithoutPdf.length) lines.push(`records without pdfs: ${corpus.recordsWithoutPdf.length} (the acquisition to-do list)`);
+  return lines.join("\n");
+}

packages/amico-run/src/papers_verb.ts

@@ -0,0 +1,22 @@
+// `amico papers` — the unified literature corpus surface (#405):
+//
+//   amico papers list [--status staged|distilled] [--tag <t>] [--platform <s>] [--q <substr>] [--json]
+//       → the corpus fold rendered: a human table by default (title · identity
+//         · status · pdf?), JSON on --json. Counts + drift (duplicates,
+//         orphans both ways) ride along — collect, unify, usable, searchable.
+//
+// Read-only (the fold never writes). $AMICO_PAPERS_VAULTS / $AMICO_PAPERS_LIBRARY
+// are the hermetic test escapes; production roots come from the studio ladder.
+import { papersList } from "./papers_list.js";
+import type { VerbResult } from "./verbs.js";
+
+export function papersVerb(argv: string[]): VerbResult {
+  const [sub, ...rest] = argv;
+  if (sub !== "list") {
+    return {
+      json: { ok: false, error: `papers: unknown subcommand '${sub ?? ""}' — usage: amico papers list [--status|--tag|--platform|--q] [--json]` },
+      code: 64,
+    };
+  }
+  return papersList(rest);
+}

packages/amico-run/src/verbs.ts

@@ -15,6 +15,7 @@
 
 import { catalogVerb } from "./catalog_verb.js";
 import { vaultVerb } from "./vault_verb.js";
+import { papersVerb } from "./papers_verb.js";
 import { deviceVerb } from "./device_verb.js";
 import { noteVerb } from "./note_verb.js";
 import { ledgerVerb } from "./ledger_verb.js";
@@ -186,4 +187,15 @@ const handoff: Verb = {
   run: (args) => handoffVerb(args),
 };
 
-export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet, spec, plan, handoff];
+// papers — the unified literature corpus (#405): one read-only view over
+// vault paper notes + the library PDF store — collected, unified, deduped
+// (reported, never merged), orphans surfaced both ways. Read-only.
+const papers: Verb = {
+  name: "papers",
+  summary: "list the unified literature corpus (vault notes + library PDFs; filters + drift)",
+  generalizes: "the literature plane's collect/unify/search surface (spec-20260817-140000)",
+  slice: "literature plane (1)",
+  run: papersVerb,
+};
+
+export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet, spec, plan, handoff, papers];