diff --git a/openkb/tree_renderer.py b/openkb/tree_renderer.py index 2424ba10..b9a03cda 100644 --- a/openkb/tree_renderer.py +++ b/openkb/tree_renderer.py @@ -15,8 +15,18 @@ def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> s return "---\n" + "\n".join(lines) + "\n---\n" -def _render_nodes_summary(nodes: list[dict], depth: int) -> str: - """Recursively render nodes for the *summary* view (summaries only).""" +def _render_nodes_summary(nodes: list[dict], depth: int, seen: dict[str, str]) -> str: + """Recursively render nodes for the *summary* view (summaries only). + + Nodes on the same source page can be handed byte-identical text by the + underlying PageIndex tree (whole-page text slicing, not a finer offset — + see PageIndex#340), which then produces duplicate LLM summaries. This + isn't confined to parent/child pairs — siblings and cousins anywhere + earlier in the document can collide too — so ``seen`` maps every summary + already rendered to the title that first produced it, and any later node + with the same summary is rendered as a short pointer back to that title + instead of repeating the block. + """ lines: list[str] = [] heading_prefix = "#" * min(depth, 6) for node in nodes: @@ -27,10 +37,16 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str: children = node.get("nodes", []) lines.append(f"{heading_prefix} {title} (pages {start}–{end})\n") - if summary: + + first_seen_title = seen.get(summary) if summary else None + if first_seen_title is not None: + lines.append(f'_(same content as "{first_seen_title}" above)_\n') + elif summary: lines.append(f"Summary: {summary}\n") + seen[summary] = title + if children: - lines.append(_render_nodes_summary(children, depth + 1)) + lines.append(_render_nodes_summary(children, depth + 1, seen)) return "\n".join(lines) @@ -44,5 +60,5 @@ def render_summary_md(tree: dict, source_name: str, doc_id: str, description: st """ frontmatter = _yaml_frontmatter(source_name, doc_id, description) structure = tree.get("structure", []) - body = _render_nodes_summary(structure, depth=1) + body = _render_nodes_summary(structure, depth=1, seen={}) return frontmatter + "\n" + body diff --git a/tests/test_tree_renderer.py b/tests/test_tree_renderer.py index 3786cfe4..86df5719 100644 --- a/tests/test_tree_renderer.py +++ b/tests/test_tree_renderer.py @@ -52,6 +52,101 @@ def test_summary_md_has_type_and_description(): assert 'full_text: "sources/my-doc.json"' in md +def test_duplicate_sibling_summaries_collapse_to_a_pointer(): + # Two sibling nodes on the same physical page can be handed the exact + # same LLM-written summary (PageIndex#340). The second occurrence should + # collapse to a pointer instead of repeating the block verbatim. + tree = { + "structure": [ + { + "title": "1.1 First item", + "start_index": 5, + "end_index": 5, + "summary": "Shared duplicate summary text.", + "nodes": [], + }, + { + "title": "1.2 Second item", + "start_index": 5, + "end_index": 5, + "summary": "Shared duplicate summary text.", + "nodes": [], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert md.count("Summary: Shared duplicate summary text.") == 1 + assert '_(same content as "1.1 First item" above)_' in md + + +def test_duplicate_summaries_collapse_across_cousins_not_just_siblings(): + # The collision isn't confined to direct siblings — a node nested under + # a different parent, seen later in document order, can repeat the same + # summary too. + tree = { + "structure": [ + { + "title": "Parent A", + "start_index": 1, + "end_index": 1, + "summary": "", + "nodes": [ + { + "title": "Child A.1", + "start_index": 1, + "end_index": 1, + "summary": "Cousin duplicate.", + "nodes": [], + } + ], + }, + { + "title": "Parent B", + "start_index": 2, + "end_index": 2, + "summary": "", + "nodes": [ + { + "title": "Child B.1", + "start_index": 2, + "end_index": 2, + "summary": "Cousin duplicate.", + "nodes": [], + } + ], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert md.count("Summary: Cousin duplicate.") == 1 + assert '_(same content as "Child A.1" above)_' in md + + +def test_distinct_summaries_are_not_collapsed(): + tree = { + "structure": [ + { + "title": "A", + "start_index": 1, + "end_index": 1, + "summary": "Summary one.", + "nodes": [], + }, + { + "title": "B", + "start_index": 2, + "end_index": 2, + "summary": "Summary two.", + "nodes": [], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert "Summary: Summary one." in md + assert "Summary: Summary two." in md + assert "same content as" not in md + + def test_summary_full_text_quoted_yaml_safe(): import yaml