From 6a0deb517c91fab01d56db783d2ca83f8752b0ad Mon Sep 17 00:00:00 2001 From: "Dana K. Williams" Date: Thu, 2 Jul 2026 23:43:36 +0000 Subject: [PATCH] fix(tree_renderer): render real per-node source text, not just the summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openkb/indexer.py already requests IndexConfig(if_add_node_text=True) and gets real per-node text back from pageindex, but the renderer only read title/summary/page range and silently discarded it — the actual document wording was unrecoverable from the rendered wiki page at all, only an LLM paraphrase. Render node["text"] as a quoted "Source text:" block after the Summary line. Also strip PageIndex's own embedded image links from that text: they point into its private .openkb/files/{doc_id}/images/ cache, not wiki/sources/images/, so they never resolved from a wiki page — the real per-page images are referenced from wiki/sources/{doc_name}.json instead. --- openkb/tree_renderer.py | 26 ++++++++++++++++++++++++- tests/test_tree_renderer.py | 39 ++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/openkb/tree_renderer.py b/openkb/tree_renderer.py index 2424ba10..32122926 100644 --- a/openkb/tree_renderer.py +++ b/openkb/tree_renderer.py @@ -2,8 +2,27 @@ from __future__ import annotations +import re + from openkb import frontmatter +# PageIndex's own include_text extraction embeds image links into its private +# cache (.openkb/files/{doc_id}/images/...), not wiki/sources/images/ where +# OpenKB's own page extraction saves them — those paths don't resolve from a +# wiki page and aren't part of the Obsidian vault. Strip them from the quoted +# source text; the real per-page images are referenced from +# wiki/sources/{doc_name}.json instead. +_PAGEINDEX_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)\n?") + + +def _strip_internal_image_refs(text: str) -> str: + return _PAGEINDEX_IMAGE_RE.sub("", text) + + +def _quote_block(text: str) -> str: + """Render ``text`` as a Markdown blockquote, one ``>`` per line.""" + return "\n".join(f"> {line}" if line else ">" for line in text.strip().splitlines()) + def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> str: """Return a YAML frontmatter block for a PageIndex wiki page.""" @@ -16,7 +35,7 @@ def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> s def _render_nodes_summary(nodes: list[dict], depth: int) -> str: - """Recursively render nodes for the *summary* view (summaries only).""" + """Recursively render nodes for the *summary* view (summary + source text).""" lines: list[str] = [] heading_prefix = "#" * min(depth, 6) for node in nodes: @@ -24,11 +43,16 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str: start = node.get("start_index", "") end = node.get("end_index", "") summary = node.get("summary", "") + text = node.get("text", "") children = node.get("nodes", []) lines.append(f"{heading_prefix} {title} (pages {start}–{end})\n") if summary: lines.append(f"Summary: {summary}\n") + stripped_text = _strip_internal_image_refs(text).strip() if text else "" + if stripped_text: + lines.append("Source text:\n") + lines.append(_quote_block(stripped_text) + "\n") if children: lines.append(_render_nodes_summary(children, depth + 1)) diff --git a/tests/test_tree_renderer.py b/tests/test_tree_renderer.py index 3786cfe4..923de7f2 100644 --- a/tests/test_tree_renderer.py +++ b/tests/test_tree_renderer.py @@ -31,12 +31,45 @@ def test_page_range_included(self, sample_tree): assert "(pages 0–120)" in output assert "(pages 121–200)" in output - def test_summary_included_not_text(self, sample_tree): + def test_summary_and_source_text_both_included(self, sample_tree): output = render_summary_md(sample_tree, "Sample Document", "doc-abc") assert "Summary: Overview of the document topic." in output assert "Summary: Historical context." in output - # Raw text should NOT appear in summary view - assert "This document introduces the core concepts of the system." not in output + # The real per-node source text is now quoted too, not just a + # paraphrase — IndexConfig(if_add_node_text=True) already fetches + # it, the old renderer just silently discarded it. + assert "Source text:" in output + assert "> This document introduces the core concepts of the system." in output + + def test_node_without_text_has_no_source_text_block(self): + tree = { + "structure": [ + {"title": "Intro", "start_index": 1, "end_index": 2, "summary": "x", "nodes": []} + ] + } + output = render_summary_md(tree, "my-doc", "doc-123") + assert "Source text:" not in output + + def test_internal_pageindex_image_refs_are_stripped_from_source_text(self): + # PageIndex's own image refs point into its private + # .openkb/files/{doc_id}/images/... cache, which never resolves from + # a wiki page, so they're stripped rather than quoted verbatim. + tree = { + "structure": [ + { + "title": "Intro", + "start_index": 1, + "end_index": 2, + "summary": "x", + "text": "Some text.\n![fig](/private/cache/img.png)\nMore text.", + "nodes": [], + } + ] + } + output = render_summary_md(tree, "my-doc", "doc-123") + assert "![fig]" not in output + assert "> Some text." in output + assert "> More text." in output def test_summary_md_has_type_and_description():