From d02f1066eae66408b7476e48a17a5ec4fbbf8311 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 29 Aug 2026 20:13:04 -0500 Subject: [PATCH 01/19] feat(core): add deterministic wiki projector Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 632 ++++++++++++++++++ .../wiki_projector/basic_projection.json | 11 + tests/indexing/test_wiki_projector.py | 258 +++++++ 3 files changed, 901 insertions(+) create mode 100644 src/basic_memory/indexing/wiki_projector.py create mode 100644 tests/fixtures/wiki_projector/basic_projection.json create mode 100644 tests/indexing/test_wiki_projector.py diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py new file mode 100644 index 000000000..aaaaeeed7 --- /dev/null +++ b/src/basic_memory/indexing/wiki_projector.py @@ -0,0 +1,632 @@ +"""Deterministic, storage-neutral planning for the Basic Memory Wiki Projector.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import StrEnum +from hashlib import sha256 +import json +from pathlib import PurePosixPath + +OKF_VERSION = "0.2" +WIKI_PROFILE = "wiki/1" +WIKI_PROJECTOR_NAME = "Basic Memory Wiki Projector" +WIKI_PROJECTOR_SOURCE = "wiki_projector" +RESERVED_WIKI_FILENAMES = frozenset({"index.md", "log.md"}) + + +class WikiProjectionReason(StrEnum): + """Why a projector run was requested.""" + + accepted_note = "accepted_note" + project_created = "project_created" + import_rebuild = "import_rebuild" + manual_rebuild = "manual_rebuild" + + +class WikiChangeOperation(StrEnum): + """Accepted note operation represented in generated Wiki logs.""" + + created = "created" + updated = "updated" + moved = "moved" + deleted = "deleted" + + +class WikiProjectionState(StrEnum): + """User-visible state derived from a projector result or run ledger.""" + + current = "current" + updating = "updating" + partial = "partial" + conflicted = "conflicted" + failed = "failed" + + +@dataclass(frozen=True, slots=True) +class WikiProjectionRequest: + """Portable request consumed by local and Cloud projector adapters.""" + + project_id: str + through_partition_position: int + projector_version: str + reason: WikiProjectionReason + requested_scopes: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.project_id.strip(): + raise ValueError("Wiki projection requires a project_id") + if self.through_partition_position < 0: + raise ValueError("Wiki projection position cannot be negative") + if not self.projector_version.strip(): + raise ValueError("Wiki projection requires a projector_version") + normalized_scopes = tuple( + sorted({_normalize_scope(scope) for scope in self.requested_scopes}) + ) + object.__setattr__(self, "requested_scopes", normalized_scopes) + + @property + def is_full_rebuild(self) -> bool: + return self.reason in { + WikiProjectionReason.import_rebuild, + WikiProjectionReason.manual_rebuild, + } + + +@dataclass(frozen=True, slots=True) +class WikiSourceNote: + """One accepted, materialized note visible to a projector snapshot.""" + + path: str + title: str + note_type: str + checksum: str + + def __post_init__(self) -> None: + object.__setattr__(self, "path", _normalize_note_path(self.path)) + if not self.title.strip(): + raise ValueError(f"Wiki source note {self.path} requires a title") + if not self.note_type.strip(): + raise ValueError(f"Wiki source note {self.path} requires a note_type") + if not self.checksum.strip(): + raise ValueError(f"Wiki source note {self.path} requires a checksum") + + +@dataclass(frozen=True, slots=True) +class WikiSourceChange: + """One accepted project-partition change used for materialization-aware logs.""" + + partition_position: int + operation: WikiChangeOperation + path: str + title: str + accepted_at: datetime + materialized: bool + source: str + previous_path: str | None = None + + def __post_init__(self) -> None: + if self.partition_position <= 0: + raise ValueError("Wiki source change position must be positive") + object.__setattr__(self, "path", _normalize_note_path(self.path)) + if self.previous_path is not None: + object.__setattr__(self, "previous_path", _normalize_note_path(self.previous_path)) + if not self.title.strip(): + raise ValueError(f"Wiki source change {self.path} requires a title") + if self.accepted_at.tzinfo is None: + raise ValueError("Wiki source change accepted_at must be timezone-aware") + if not self.source.strip(): + raise ValueError("Wiki source change requires a source") + + +@dataclass(frozen=True, slots=True) +class WikiReservedDocument: + """Current accepted state for a path reserved to the Wiki Projector.""" + + path: str + checksum: str + content: bytes + projector_owned: bool + + def __post_init__(self) -> None: + normalized_path = _normalize_note_path(self.path) + if PurePosixPath(normalized_path).name.lower() not in RESERVED_WIKI_FILENAMES: + raise ValueError(f"Wiki reserved document has non-reserved path: {self.path}") + if not self.checksum.strip(): + raise ValueError(f"Wiki reserved document {self.path} requires a checksum") + object.__setattr__(self, "path", normalized_path) + + +@dataclass(frozen=True, slots=True) +class WikiProjectionSnapshot: + """Complete deterministic input needed to plan one projector run.""" + + project_id: str + project_name: str + current_output_watermark: int + source_accepted_at: datetime + notes: tuple[WikiSourceNote, ...] + changes: tuple[WikiSourceChange, ...] + reserved_documents: tuple[WikiReservedDocument, ...] = () + + def __post_init__(self) -> None: + if not self.project_id.strip(): + raise ValueError("Wiki projection snapshot requires a project_id") + if not self.project_name.strip(): + raise ValueError("Wiki projection snapshot requires a project_name") + if self.current_output_watermark < 0: + raise ValueError("Wiki output watermark cannot be negative") + if self.source_accepted_at.tzinfo is None: + raise ValueError("Wiki snapshot source_accepted_at must be timezone-aware") + _require_unique_paths(self.notes, label="source note") + _require_unique_paths(self.reserved_documents, label="reserved document") + positions = [change.partition_position for change in self.changes] + if len(positions) != len(set(positions)): + raise ValueError("Wiki source changes require unique partition positions") + + +@dataclass(frozen=True, slots=True) +class WikiProjectionWrite: + """Checksum-protected canonical Markdown write planned for an adapter.""" + + path: str + content: bytes + checksum: str + expected_checksum: str | None + + +@dataclass(frozen=True, slots=True) +class WikiProjectionConflict: + """Reserved path the projector cannot safely claim or replace.""" + + path: str + reason: str + + +@dataclass(frozen=True, slots=True) +class WikiProjectionResult: + """Portable outcome recorded by local and Cloud run ledgers.""" + + source_watermark: int + output_watermark: int + created: int + updated: int + unchanged: int + conflicts: tuple[WikiProjectionConflict, ...] + warnings: tuple[str, ...] + pending_materialization: tuple[int, ...] + + @property + def state(self) -> WikiProjectionState: + if self.conflicts: + return WikiProjectionState.conflicted + if self.pending_materialization: + return WikiProjectionState.partial + if self.output_watermark < self.source_watermark: + return WikiProjectionState.updating + return WikiProjectionState.current + + +@dataclass(frozen=True, slots=True) +class WikiProjectionPlan: + """Pure projection result plus writes for a runtime adapter to execute.""" + + request: WikiProjectionRequest + writes: tuple[WikiProjectionWrite, ...] + unchanged_paths: tuple[str, ...] + result: WikiProjectionResult + + +def affected_wiki_scopes(*paths: str | None) -> tuple[str, ...]: + """Return root and every ancestor directory affected by note paths.""" + scopes = {""} + for path in paths: + if path is None: + continue + parent = PurePosixPath(_normalize_note_path(path)).parent + while parent != PurePosixPath("."): + scopes.add(parent.as_posix()) + parent = parent.parent + return tuple(sorted(scopes)) + + +def plan_wiki_projection( + request: WikiProjectionRequest, + snapshot: WikiProjectionSnapshot, +) -> WikiProjectionPlan: + """Plan deterministic OKF index/log writes without performing I/O.""" + if request.project_id != snapshot.project_id: + raise ValueError("Wiki projection request and snapshot project_id differ") + if request.through_partition_position < snapshot.current_output_watermark: + raise ValueError("Wiki projection request is older than the current output watermark") + + changes = tuple( + sorted( + ( + change + for change in snapshot.changes + if change.partition_position <= request.through_partition_position + and not _is_projector_change(change) + ), + key=lambda change: change.partition_position, + ) + ) + pending = tuple( + change.partition_position + for change in changes + if not change.materialized and change.partition_position > snapshot.current_output_watermark + ) + if pending: + warning = ( + "Projection deferred until accepted note positions are materialized: " + + ", ".join(str(position) for position in pending) + ) + return WikiProjectionPlan( + request=request, + writes=(), + unchanged_paths=(), + result=WikiProjectionResult( + source_watermark=request.through_partition_position, + output_watermark=snapshot.current_output_watermark, + created=0, + updated=0, + unchanged=0, + conflicts=(), + warnings=(warning,), + pending_materialization=pending, + ), + ) + + # A projector-only replay advances the ledger without rewriting its own + # generated notes. Full rebuilds and project creation remain explicit work. + new_changes = tuple( + change + for change in changes + if change.partition_position > snapshot.current_output_watermark + ) + if ( + request.reason == WikiProjectionReason.accepted_note + and not new_changes + and snapshot.reserved_documents + ): + return WikiProjectionPlan( + request=request, + writes=(), + unchanged_paths=tuple( + sorted(document.path for document in snapshot.reserved_documents) + ), + result=WikiProjectionResult( + source_watermark=request.through_partition_position, + output_watermark=request.through_partition_position, + created=0, + updated=0, + unchanged=len(snapshot.reserved_documents), + conflicts=(), + warnings=(), + pending_materialization=(), + ), + ) + + scopes = _projection_scopes(request, snapshot, new_changes) + existing_by_path = {document.path: document for document in snapshot.reserved_documents} + notes = tuple( + note + for note in snapshot.notes + if PurePosixPath(note.path).name.lower() not in RESERVED_WIKI_FILENAMES + ) + rendered: dict[str, bytes] = {} + for scope in scopes: + rendered[_reserved_path(scope, "index.md")] = _render_index( + snapshot=snapshot, + notes=notes, + scope=scope, + source_watermark=request.through_partition_position, + ) + rendered[_reserved_path(scope, "log.md")] = _render_log( + snapshot=snapshot, + changes=changes, + scope=scope, + source_watermark=request.through_partition_position, + ) + + conflicts = tuple( + WikiProjectionConflict( + path=path, + reason="reserved path is not owned by the Wiki Projector", + ) + for path in sorted(rendered) + if (existing := existing_by_path.get(path)) is not None and not existing.projector_owned + ) + if conflicts: + # Indexes and logs describe one project watermark. Writing only the + # unblocked paths would publish a mixed projection that no ledger + # watermark could honestly represent, so conflict is all-or-nothing. + return WikiProjectionPlan( + request=request, + writes=(), + unchanged_paths=(), + result=WikiProjectionResult( + source_watermark=request.through_partition_position, + output_watermark=snapshot.current_output_watermark, + created=0, + updated=0, + unchanged=0, + conflicts=conflicts, + warnings=(), + pending_materialization=(), + ), + ) + + writes: list[WikiProjectionWrite] = [] + unchanged_paths: list[str] = [] + created = 0 + updated = 0 + for path, content in sorted(rendered.items()): + existing = existing_by_path.get(path) + if existing is not None and existing.content == content: + unchanged_paths.append(path) + continue + writes.append( + WikiProjectionWrite( + path=path, + content=content, + checksum=sha256(content).hexdigest(), + expected_checksum=existing.checksum if existing is not None else None, + ) + ) + if existing is None: + created += 1 + else: + updated += 1 + + return WikiProjectionPlan( + request=request, + writes=tuple(writes), + unchanged_paths=tuple(unchanged_paths), + result=WikiProjectionResult( + source_watermark=request.through_partition_position, + output_watermark=request.through_partition_position, + created=created, + updated=updated, + unchanged=len(unchanged_paths), + conflicts=(), + warnings=(), + pending_materialization=(), + ), + ) + + +def _projection_scopes( + request: WikiProjectionRequest, + snapshot: WikiProjectionSnapshot, + new_changes: tuple[WikiSourceChange, ...], +) -> tuple[str, ...]: + if request.is_full_rebuild: + paths = [note.path for note in snapshot.notes] + return affected_wiki_scopes(*paths) + if request.requested_scopes: + scopes = {""} + for requested_scope in request.requested_scopes: + scope = PurePosixPath(requested_scope) + while scope != PurePosixPath("."): + scopes.add(scope.as_posix()) + scope = scope.parent + return tuple(sorted(scopes)) + paths = [change.path for change in new_changes] + paths.extend(change.previous_path for change in new_changes if change.previous_path is not None) + return affected_wiki_scopes(*paths) + + +def _render_index( + *, + snapshot: WikiProjectionSnapshot, + notes: tuple[WikiSourceNote, ...], + scope: str, + source_watermark: int, +) -> bytes: + direct_notes = sorted( + (note for note in notes if _parent_scope(note.path) == scope), + key=lambda note: (note.title.casefold(), note.path.casefold()), + ) + child_scope_set: set[str] = set() + for note in notes: + if not _is_descendant(note.path, scope): + continue + child_scope = _direct_child_scope(scope, note.path) + if child_scope is not None: + child_scope_set.add(child_scope) + child_scopes = sorted(child_scope_set) + title = snapshot.project_name if not scope else _display_name(PurePosixPath(scope).name) + body: list[str] = [f"# {title}", ""] + if child_scopes: + body.extend(["## Sections", ""]) + body.extend( + f"- [[{child_scope}/index|{_display_name(PurePosixPath(child_scope).name)}]]" + for child_scope in child_scopes + ) + body.append("") + if direct_notes: + body.extend(["## Notes", ""]) + body.extend( + f"- [[{_without_markdown_suffix(note.path)}|{note.title}]]" for note in direct_notes + ) + body.append("") + if not child_scopes and not direct_notes: + body.extend(["No concepts have been projected into this scope yet.", ""]) + return _render_document( + note_type="Index", + title=title, + source_watermark=source_watermark, + generated_at=snapshot.source_accepted_at, + body="\n".join(body), + include_okf_version=not scope, + ) + + +def _render_log( + *, + snapshot: WikiProjectionSnapshot, + changes: tuple[WikiSourceChange, ...], + scope: str, + source_watermark: int, +) -> bytes: + relevant = tuple( + sorted( + ( + change + for change in changes + if change.materialized + and ( + _is_descendant(change.path, scope) + or ( + change.previous_path is not None + and _is_descendant(change.previous_path, scope) + ) + ) + ), + key=lambda change: change.partition_position, + reverse=True, + ) + ) + title = ( + f"{snapshot.project_name} log" + if not scope + else f"{_display_name(PurePosixPath(scope).name)} log" + ) + body: list[str] = [f"# {title}", ""] + if relevant: + body.extend(_render_log_entry(change) for change in relevant) + body.append("") + else: + body.extend(["No accepted materialized changes have been recorded yet.", ""]) + return _render_document( + note_type="Log", + title=title, + source_watermark=source_watermark, + generated_at=snapshot.source_accepted_at, + body="\n".join(body), + include_okf_version=False, + ) + + +def _render_log_entry(change: WikiSourceChange) -> str: + timestamp = _isoformat_utc(change.accepted_at) + match change.operation: + case WikiChangeOperation.created: + description = f"Created [[{_without_markdown_suffix(change.path)}|{change.title}]]" + case WikiChangeOperation.updated: + description = f"Updated [[{_without_markdown_suffix(change.path)}|{change.title}]]" + case WikiChangeOperation.moved: + if change.previous_path is None: + raise ValueError("Moved Wiki change requires previous_path") + description = ( + f"Moved `{change.previous_path}` to " + f"[[{_without_markdown_suffix(change.path)}|{change.title}]]" + ) + case WikiChangeOperation.deleted: + description = f"Deleted `{change.path}`" + return f"- {timestamp} — {description}" + + +def _render_document( + *, + note_type: str, + title: str, + source_watermark: int, + generated_at: datetime, + body: str, + include_okf_version: bool, +) -> bytes: + frontmatter = ["---", f"type: {note_type}"] + if include_okf_version: + frontmatter.append(f'okf_version: "{OKF_VERSION}"') + frontmatter.extend( + [ + f"title: {json.dumps(title, ensure_ascii=False)}", + "generated:", + f" by: {WIKI_PROJECTOR_NAME}", + f" at: {json.dumps(_isoformat_utc(generated_at))}", + "bm:", + f" profile: {WIKI_PROFILE}", + f' source_watermark: "{source_watermark}"', + "---", + body, + ] + ) + return ("\n".join(frontmatter).rstrip() + "\n").encode("utf-8") + + +def _is_projector_change(change: WikiSourceChange) -> bool: + return ( + change.source == WIKI_PROJECTOR_SOURCE + and PurePosixPath(change.path).name.lower() in RESERVED_WIKI_FILENAMES + ) + + +def _normalize_note_path(path: str) -> str: + normalized = _normalize_relative_path(path) + if not normalized or PurePosixPath(normalized).suffix.lower() != ".md": + raise ValueError(f"Wiki note path must be project-relative Markdown: {path}") + return normalized + + +def _normalize_scope(scope: str) -> str: + return _normalize_relative_path(scope) + + +def _normalize_relative_path(path: str) -> str: + candidate = path.strip().replace("\\", "/") + if candidate.startswith("/"): + raise ValueError(f"Wiki path must be project-relative and normalized: {path}") + candidate = candidate.strip("/") + if not candidate: + return "" + parsed = PurePosixPath(candidate) + if parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts): + raise ValueError(f"Wiki path must be project-relative and normalized: {path}") + return parsed.as_posix() + + +def _require_unique_paths(values: tuple[object, ...], *, label: str) -> None: + paths = [getattr(value, "path") for value in values] + if len(paths) != len(set(paths)): + raise ValueError(f"Wiki projection snapshot has duplicate {label} paths") + + +def _reserved_path(scope: str, filename: str) -> str: + return f"{scope}/{filename}" if scope else filename + + +def _parent_scope(path: str) -> str: + parent = PurePosixPath(path).parent + return "" if parent == PurePosixPath(".") else parent.as_posix() + + +def _is_descendant(path: str, scope: str) -> bool: + if not scope: + return True + return path == scope or path.startswith(f"{scope}/") + + +def _direct_child_scope(scope: str, note_path: str) -> str | None: + note_parent = _parent_scope(note_path) + if not note_parent or note_parent == scope: + return None + prefix = f"{scope}/" if scope else "" + if not note_parent.startswith(prefix): + return None + child_name = note_parent[len(prefix) :].split("/", maxsplit=1)[0] + return f"{scope}/{child_name}" if scope else child_name + + +def _display_name(value: str) -> str: + return value.replace("-", " ").replace("_", " ").strip().title() + + +def _without_markdown_suffix(path: str) -> str: + return path[:-3] if path.lower().endswith(".md") else path + + +def _isoformat_utc(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/tests/fixtures/wiki_projector/basic_projection.json b/tests/fixtures/wiki_projector/basic_projection.json new file mode 100644 index 000000000..0ef9090d1 --- /dev/null +++ b/tests/fixtures/wiki_projector/basic_projection.json @@ -0,0 +1,11 @@ +{ + "contract_version": "wiki/1.0.0", + "project_id": "project-88", + "through_partition_position": 3, + "expected_sha256": { + "guides/index.md": "c6481bd17c663a3c2595cb35cd22a03da46c6d55465a274482a91363b3af244a", + "guides/log.md": "a3bf317e661d40a156a7481478938b8a652d4a69184b06cfa8c1b8b5355307fb", + "index.md": "33a4af87c4c1a3d4e128f780e67a8a7084aa1796118fb7a5334bbec4a4e0728d", + "log.md": "8e21ce176e930556f6a39f30412af7c488f9724b4118d2946879b72d0eb6c2f3" + } +} diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py new file mode 100644 index 000000000..5beae52ae --- /dev/null +++ b/tests/indexing/test_wiki_projector.py @@ -0,0 +1,258 @@ +"""Deterministic Wiki Projector contract and byte-output tests.""" + +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path + +import pytest + +from basic_memory.indexing.wiki_projector import ( + WikiChangeOperation, + WikiProjectionReason, + WikiProjectionRequest, + WikiProjectionSnapshot, + WikiProjectionState, + WikiReservedDocument, + WikiSourceChange, + WikiSourceNote, + affected_wiki_scopes, + plan_wiki_projection, +) + +ACCEPTED_AT = datetime(2026, 8, 29, 18, 30, tzinfo=timezone.utc) + + +def _request( + *, + position: int = 3, + reason: WikiProjectionReason = WikiProjectionReason.accepted_note, + scopes: tuple[str, ...] = ("guides",), +) -> WikiProjectionRequest: + return WikiProjectionRequest( + project_id="project-88", + through_partition_position=position, + projector_version="wiki/1.0.0", + reason=reason, + requested_scopes=scopes, + ) + + +def _snapshot( + *, + output_watermark: int = 2, + materialized: bool = True, + reserved_documents: tuple[WikiReservedDocument, ...] = (), +) -> WikiProjectionSnapshot: + return WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=output_watermark, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="overview.md", + title="Overview", + note_type="Note", + checksum="overview-checksum", + ), + WikiSourceNote( + path="guides/setup.md", + title="Setup", + note_type="Guide", + checksum="setup-checksum", + ), + WikiSourceNote( + path="guides/deep/details.md", + title="Details", + note_type="Guide", + checksum="details-checksum", + ), + ), + changes=( + WikiSourceChange( + partition_position=3, + operation=WikiChangeOperation.updated, + path="guides/setup.md", + title="Setup", + accepted_at=ACCEPTED_AT, + materialized=materialized, + source="web", + ), + ), + reserved_documents=reserved_documents, + ) + + +def _reserved(path: str, content: bytes, *, owned: bool = True) -> WikiReservedDocument: + return WikiReservedDocument( + path=path, + checksum=sha256(content).hexdigest(), + content=content, + projector_owned=owned, + ) + + +def test_affected_scopes_include_root_and_move_ancestors() -> None: + assert affected_wiki_scopes("guides/old/setup.md", "reference/new/setup.md") == ( + "", + "guides", + "guides/old", + "reference", + "reference/new", + ) + + +def test_projection_renders_root_and_affected_directory_indexes_and_logs() -> None: + plan = plan_wiki_projection(_request(), _snapshot()) + + assert [write.path for write in plan.writes] == [ + "guides/index.md", + "guides/log.md", + "index.md", + "log.md", + ] + rendered = {write.path: write.content.decode() for write in plan.writes} + assert "[[guides/deep/index|Deep]]" in rendered["guides/index.md"] + assert "[[guides/setup|Setup]]" in rendered["guides/index.md"] + assert "[[guides/index|Guides]]" in rendered["index.md"] + assert "[[overview|Overview]]" in rendered["index.md"] + assert "Updated [[guides/setup|Setup]]" in rendered["guides/log.md"] + assert plan.result.source_watermark == 3 + assert plan.result.output_watermark == 3 + assert plan.result.created == 4 + assert plan.result.state == WikiProjectionState.current + + +def test_projection_bytes_match_the_shared_contract_fixture() -> None: + fixture_path = ( + Path(__file__).parents[1] / "fixtures" / "wiki_projector" / "basic_projection.json" + ) + fixture = json.loads(fixture_path.read_text()) + + plan = plan_wiki_projection(_request(), _snapshot()) + + assert fixture["contract_version"] == plan.request.projector_version + assert fixture["project_id"] == plan.request.project_id + assert fixture["through_partition_position"] == plan.request.through_partition_position + assert fixture["expected_sha256"] == {write.path: write.checksum for write in plan.writes} + + +def test_projection_is_a_byte_identical_noop_at_the_same_watermark() -> None: + first = plan_wiki_projection(_request(), _snapshot()) + existing = tuple(_reserved(write.path, write.content) for write in first.writes) + + replay = plan_wiki_projection( + _request(), + _snapshot(output_watermark=3, reserved_documents=existing), + ) + + assert replay.writes == () + assert replay.unchanged_paths == tuple(write.path for write in first.writes) + assert replay.result.unchanged == 4 + assert replay.result.state == WikiProjectionState.current + + +def test_pending_materialization_defers_all_bytes_without_advancing_output() -> None: + plan = plan_wiki_projection(_request(), _snapshot(materialized=False)) + + assert plan.writes == () + assert plan.result.output_watermark == 2 + assert plan.result.pending_materialization == (3,) + assert plan.result.state == WikiProjectionState.partial + + +def test_user_claimed_reserved_path_is_a_conflict_not_a_write() -> None: + claimed = _reserved("guides/index.md", b"# User index\n", owned=False) + + plan = plan_wiki_projection( + _request(), + _snapshot(reserved_documents=(claimed,)), + ) + + assert plan.writes == () + assert plan.result.conflicts[0].path == "guides/index.md" + assert plan.result.output_watermark == 2 + assert plan.result.state == WikiProjectionState.conflicted + + +def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: + existing = _reserved("index.md", b"existing\n") + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=3, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=4, + operation=WikiChangeOperation.updated, + path="index.md", + title="Project 88", + accepted_at=ACCEPTED_AT, + materialized=True, + source="wiki_projector", + ), + ), + reserved_documents=(existing,), + ) + + plan = plan_wiki_projection(_request(position=4, scopes=()), snapshot) + + assert plan.writes == () + assert plan.result.output_watermark == 4 + assert plan.result.state == WikiProjectionState.current + + +def test_full_rebuild_covers_every_note_directory() -> None: + request = _request( + reason=WikiProjectionReason.import_rebuild, + scopes=(), + ) + + plan = plan_wiki_projection(request, _snapshot()) + + assert {write.path for write in plan.writes} == { + "index.md", + "log.md", + "guides/index.md", + "guides/log.md", + "guides/deep/index.md", + "guides/deep/log.md", + } + + +def test_moved_change_requires_previous_path() -> None: + with pytest.raises(ValueError, match="requires previous_path"): + plan_wiki_projection( + _request(), + WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=2, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=3, + operation=WikiChangeOperation.moved, + path="guides/new.md", + title="Moved", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ), + ) + + +def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: + with pytest.raises(ValueError, match="project-relative"): + WikiSourceNote( + path="/outside.md", + title="Outside", + note_type="Note", + checksum="checksum", + ) From 8c6ad35ac2715be3444372457946332cf74d4f76 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 15:27:20 -0500 Subject: [PATCH 02/19] fix wiki projector output safety Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 59 ++++++++++++---- tests/indexing/test_wiki_projector.py | 75 +++++++++++++++++++++ 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index aaaaeeed7..9f6a50bac 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -7,7 +7,7 @@ from enum import StrEnum from hashlib import sha256 import json -from pathlib import PurePosixPath +from pathlib import PurePosixPath, PureWindowsPath OKF_VERSION = "0.2" WIKI_PROFILE = "wiki/1" @@ -427,7 +427,12 @@ def _render_index( ) -> bytes: direct_notes = sorted( (note for note in notes if _parent_scope(note.path) == scope), - key=lambda note: (note.title.casefold(), note.path.casefold()), + key=lambda note: ( + note.title.casefold(), + note.path.casefold(), + note.title, + note.path, + ), ) child_scope_set: set[str] = set() for note in notes: @@ -438,18 +443,23 @@ def _render_index( child_scope_set.add(child_scope) child_scopes = sorted(child_scope_set) title = snapshot.project_name if not scope else _display_name(PurePosixPath(scope).name) - body: list[str] = [f"# {title}", ""] + body: list[str] = [f"# {_escape_generated_markdown_text(title)}", ""] if child_scopes: body.extend(["## Sections", ""]) body.extend( - f"- [[{child_scope}/index|{_display_name(PurePosixPath(child_scope).name)}]]" + "- " + f"[[{_escape_generated_markdown_text(f'{child_scope}/index')}|" + f"{_escape_generated_markdown_text(_display_name(PurePosixPath(child_scope).name))}]]" for child_scope in child_scopes ) body.append("") if direct_notes: body.extend(["## Notes", ""]) body.extend( - f"- [[{_without_markdown_suffix(note.path)}|{note.title}]]" for note in direct_notes + "- " + f"[[{_escape_generated_markdown_text(_without_markdown_suffix(note.path))}|" + f"{_escape_generated_markdown_text(note.title)}]]" + for note in direct_notes ) body.append("") if not child_scopes and not direct_notes: @@ -494,7 +504,7 @@ def _render_log( if not scope else f"{_display_name(PurePosixPath(scope).name)} log" ) - body: list[str] = [f"# {title}", ""] + body: list[str] = [f"# {_escape_generated_markdown_text(title)}", ""] if relevant: body.extend(_render_log_entry(change) for change in relevant) body.append("") @@ -512,20 +522,22 @@ def _render_log( def _render_log_entry(change: WikiSourceChange) -> str: timestamp = _isoformat_utc(change.accepted_at) + path = _escape_generated_markdown_text(_without_markdown_suffix(change.path)) + title = _escape_generated_markdown_text(change.title) match change.operation: case WikiChangeOperation.created: - description = f"Created [[{_without_markdown_suffix(change.path)}|{change.title}]]" + description = f"Created [[{path}|{title}]]" case WikiChangeOperation.updated: - description = f"Updated [[{_without_markdown_suffix(change.path)}|{change.title}]]" + description = f"Updated [[{path}|{title}]]" case WikiChangeOperation.moved: if change.previous_path is None: raise ValueError("Moved Wiki change requires previous_path") description = ( - f"Moved `{change.previous_path}` to " - f"[[{_without_markdown_suffix(change.path)}|{change.title}]]" + f"Moved `{_escape_generated_markdown_text(change.previous_path)}` to " + f"[[{path}|{title}]]" ) case WikiChangeOperation.deleted: - description = f"Deleted `{change.path}`" + description = f"Deleted `{_escape_generated_markdown_text(change.path)}`" return f"- {timestamp} — {description}" @@ -576,9 +588,11 @@ def _normalize_scope(scope: str) -> str: def _normalize_relative_path(path: str) -> str: - candidate = path.strip().replace("\\", "/") - if candidate.startswith("/"): + accepted_path = path.strip() + windows_path = PureWindowsPath(accepted_path) + if accepted_path.startswith(("/", "\\")) or windows_path.drive or windows_path.is_absolute(): raise ValueError(f"Wiki path must be project-relative and normalized: {path}") + candidate = accepted_path.replace("\\", "/") candidate = candidate.strip("/") if not candidate: return "" @@ -588,6 +602,25 @@ def _normalize_relative_path(path: str) -> str: return parsed.as_posix() +def _escape_generated_markdown_text(value: str) -> str: + """Keep snapshot metadata from changing generated Markdown structure.""" + return value.translate( + str.maketrans( + { + "\r": " ", + "\n": " ", + "\\": "\", + "[": "[", + "]": "]", + "|": "|", + "`": "`", + "<": "<", + ">": ">", + } + ) + ) + + def _require_unique_paths(values: tuple[object, ...], *, label: str) -> None: paths = [getattr(value, "path") for value in values] if len(paths) != len(set(paths)): diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 5beae52ae..2cf4d2861 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -256,3 +256,78 @@ def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: note_type="Note", checksum="checksum", ) + + +@pytest.mark.parametrize("path", ("C:/outside.md", "C:\\outside.md")) +def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> None: + with pytest.raises(ValueError, match="project-relative"): + WikiSourceNote( + path=path, + title="Outside", + note_type="Note", + checksum="checksum", + ) + + +def test_projection_order_is_deterministic_for_case_only_names() -> None: + notes = ( + WikiSourceNote(path="foo.md", title="same", note_type="Note", checksum="lower"), + WikiSourceNote(path="Foo.md", title="Same", note_type="Note", checksum="upper"), + ) + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=notes, + changes=(), + ) + reverse_snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=tuple(reversed(notes)), + changes=(), + ) + + first = plan_wiki_projection(_request(position=0, scopes=()), snapshot) + second = plan_wiki_projection(_request(position=0, scopes=()), reverse_snapshot) + + assert first.writes == second.writes + + +def test_projection_escapes_dynamic_markdown_structure() -> None: + injected_title = "Bad]]\n- relates_to [[evil" + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name=f"Project\n{injected_title}", + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="unsafe]target.md", + title=injected_title, + note_type="Note", + checksum="unsafe", + ), + ), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.updated, + path="unsafe]target.md", + title=injected_title, + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection(_request(position=1, scopes=()), snapshot) + rendered = {write.path: write.content.decode() for write in plan.writes} + + assert "\n- relates_to [[evil" not in rendered["index.md"] + assert "\n- relates_to [[evil" not in rendered["log.md"] + assert "[[unsafe]target|Bad]] - relates_to [[evil]]" in rendered["index.md"] From 475f2f3b494361c9590a62f667a51bc8f2cdea39 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 15:37:03 -0500 Subject: [PATCH 03/19] fix wiki projector rebuild edge cases Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 22 ++++-- tests/indexing/test_wiki_projector.py | 81 ++++++++++++++++++++- 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 9f6a50bac..52aef87a4 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -309,7 +309,9 @@ def plan_wiki_projection( ) scopes = _projection_scopes(request, snapshot, new_changes) - existing_by_path = {document.path: document for document in snapshot.reserved_documents} + existing_by_path = { + document.path.casefold(): document for document in snapshot.reserved_documents + } notes = tuple( note for note in snapshot.notes @@ -336,7 +338,8 @@ def plan_wiki_projection( reason="reserved path is not owned by the Wiki Projector", ) for path in sorted(rendered) - if (existing := existing_by_path.get(path)) is not None and not existing.projector_owned + if (existing := existing_by_path.get(path.casefold())) is not None + and not existing.projector_owned ) if conflicts: # Indexes and logs describe one project watermark. Writing only the @@ -363,7 +366,7 @@ def plan_wiki_projection( created = 0 updated = 0 for path, content in sorted(rendered.items()): - existing = existing_by_path.get(path) + existing = existing_by_path.get(path.casefold()) if existing is not None and existing.content == content: unchanged_paths.append(path) continue @@ -404,6 +407,11 @@ def _projection_scopes( ) -> tuple[str, ...]: if request.is_full_rebuild: paths = [note.path for note in snapshot.notes] + paths.extend(document.path for document in snapshot.reserved_documents) + paths.extend(change.path for change in snapshot.changes) + paths.extend( + change.previous_path for change in snapshot.changes if change.previous_path is not None + ) return affected_wiki_scopes(*paths) if request.requested_scopes: scopes = {""} @@ -448,7 +456,7 @@ def _render_index( body.extend(["## Sections", ""]) body.extend( "- " - f"[[{_escape_generated_markdown_text(f'{child_scope}/index')}|" + f"[[{child_scope}/index|" f"{_escape_generated_markdown_text(_display_name(PurePosixPath(child_scope).name))}]]" for child_scope in child_scopes ) @@ -457,7 +465,7 @@ def _render_index( body.extend(["## Notes", ""]) body.extend( "- " - f"[[{_escape_generated_markdown_text(_without_markdown_suffix(note.path))}|" + f"[[{_without_markdown_suffix(note.path)}|" f"{_escape_generated_markdown_text(note.title)}]]" for note in direct_notes ) @@ -522,7 +530,7 @@ def _render_log( def _render_log_entry(change: WikiSourceChange) -> str: timestamp = _isoformat_utc(change.accepted_at) - path = _escape_generated_markdown_text(_without_markdown_suffix(change.path)) + path = _without_markdown_suffix(change.path) title = _escape_generated_markdown_text(change.title) match change.operation: case WikiChangeOperation.created: @@ -580,6 +588,8 @@ def _normalize_note_path(path: str) -> str: normalized = _normalize_relative_path(path) if not normalized or PurePosixPath(normalized).suffix.lower() != ".md": raise ValueError(f"Wiki note path must be project-relative Markdown: {path}") + if any(character in normalized for character in "\r\n[]|`<>"): + raise ValueError(f"Wiki note path contains unsupported Markdown delimiters: {path}") return normalized diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 2cf4d2861..14914162f 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -176,6 +176,20 @@ def test_user_claimed_reserved_path_is_a_conflict_not_a_write() -> None: assert plan.result.state == WikiProjectionState.conflicted +def test_user_claimed_reserved_path_matches_case_insensitively() -> None: + claimed = _reserved("guides/Index.md", b"# User index\n", owned=False) + + plan = plan_wiki_projection( + _request(), + _snapshot(reserved_documents=(claimed,)), + ) + + assert plan.writes == () + assert plan.result.conflicts[0].path == "guides/index.md" + assert plan.result.output_watermark == 2 + assert plan.result.state == WikiProjectionState.conflicted + + def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: existing = _reserved("index.md", b"existing\n") snapshot = WikiProjectionSnapshot( @@ -223,6 +237,46 @@ def test_full_rebuild_covers_every_note_directory() -> None: } +def test_full_rebuild_covers_orphaned_reserved_document_scopes() -> None: + request = _request( + reason=WikiProjectionReason.manual_rebuild, + scopes=(), + ) + orphaned_index = _reserved("orphaned/index.md", b"# Stale index\n") + orphaned_log = _reserved("orphaned/log.md", b"# Stale log\n") + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=2, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=3, + operation=WikiChangeOperation.deleted, + path="orphaned/last-note.md", + title="Last note", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + reserved_documents=(orphaned_index, orphaned_log), + ) + + plan = plan_wiki_projection(request, snapshot) + + assert {write.path for write in plan.writes} == { + "index.md", + "log.md", + "orphaned/index.md", + "orphaned/log.md", + } + rendered = {write.path: write.content.decode() for write in plan.writes} + assert "No concepts have been projected" in rendered["orphaned/index.md"] + assert "Deleted `orphaned/last-note.md`" in rendered["orphaned/log.md"] + + def test_moved_change_requires_previous_path() -> None: with pytest.raises(ValueError, match="requires previous_path"): plan_wiki_projection( @@ -269,6 +323,27 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> ) +@pytest.mark.parametrize( + "path", + ( + "notes/closing].md", + "notes/opening[.md", + "notes/alias|target.md", + "notes/code`span.md", + "notes/html None: + with pytest.raises(ValueError, match="unsupported Markdown delimiters"): + WikiSourceNote( + path=path, + title="Unsupported", + note_type="Note", + checksum="checksum", + ) + + def test_projection_order_is_deterministic_for_case_only_names() -> None: notes = ( WikiSourceNote(path="foo.md", title="same", note_type="Note", checksum="lower"), @@ -306,7 +381,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: source_accepted_at=ACCEPTED_AT, notes=( WikiSourceNote( - path="unsafe]target.md", + path="safe-target.md", title=injected_title, note_type="Note", checksum="unsafe", @@ -316,7 +391,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: WikiSourceChange( partition_position=1, operation=WikiChangeOperation.updated, - path="unsafe]target.md", + path="safe-target.md", title=injected_title, accepted_at=ACCEPTED_AT, materialized=True, @@ -330,4 +405,4 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: assert "\n- relates_to [[evil" not in rendered["index.md"] assert "\n- relates_to [[evil" not in rendered["log.md"] - assert "[[unsafe]target|Bad]] - relates_to [[evil]]" in rendered["index.md"] + assert "[[safe-target|Bad]] - relates_to [[evil]]" in rendered["index.md"] From 53062d3e72ecfe661d9915ace31c82ea15afb9cd Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 15:48:45 -0500 Subject: [PATCH 04/19] Harden wiki projector contract coverage Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 19 +- tests/indexing/test_wiki_projector.py | 220 ++++++++++++++++++++ 2 files changed, 234 insertions(+), 5 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 52aef87a4..5be370a3f 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -160,7 +160,11 @@ def __post_init__(self) -> None: if self.source_accepted_at.tzinfo is None: raise ValueError("Wiki snapshot source_accepted_at must be timezone-aware") _require_unique_paths(self.notes, label="source note") - _require_unique_paths(self.reserved_documents, label="reserved document") + _require_unique_paths( + self.reserved_documents, + label="reserved document", + case_sensitive=False, + ) positions = [change.partition_position for change in self.changes] if len(positions) != len(set(positions)): raise ValueError("Wiki source changes require unique partition positions") @@ -588,7 +592,7 @@ def _normalize_note_path(path: str) -> str: normalized = _normalize_relative_path(path) if not normalized or PurePosixPath(normalized).suffix.lower() != ".md": raise ValueError(f"Wiki note path must be project-relative Markdown: {path}") - if any(character in normalized for character in "\r\n[]|`<>"): + if "::" in normalized or any(character in normalized for character in "\r\n[]|`<>"): raise ValueError(f"Wiki note path contains unsupported Markdown delimiters: {path}") return normalized @@ -631,8 +635,15 @@ def _escape_generated_markdown_text(value: str) -> str: ) -def _require_unique_paths(values: tuple[object, ...], *, label: str) -> None: +def _require_unique_paths( + values: tuple[object, ...], + *, + label: str, + case_sensitive: bool = True, +) -> None: paths = [getattr(value, "path") for value in values] + if not case_sensitive: + paths = [path.casefold() for path in paths] if len(paths) != len(set(paths)): raise ValueError(f"Wiki projection snapshot has duplicate {label} paths") @@ -657,8 +668,6 @@ def _direct_child_scope(scope: str, note_path: str) -> str | None: if not note_parent or note_parent == scope: return None prefix = f"{scope}/" if scope else "" - if not note_parent.startswith(prefix): - return None child_name = note_parent[len(prefix) :].split("/", maxsplit=1)[0] return f"{scope}/{child_name}" if scope else child_name diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 14914162f..ca322b9f6 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -1,5 +1,6 @@ """Deterministic Wiki Projector contract and byte-output tests.""" +from dataclasses import replace from datetime import datetime, timezone from hashlib import sha256 import json @@ -11,6 +12,7 @@ WikiChangeOperation, WikiProjectionReason, WikiProjectionRequest, + WikiProjectionResult, WikiProjectionSnapshot, WikiProjectionState, WikiReservedDocument, @@ -103,6 +105,143 @@ def test_affected_scopes_include_root_and_move_ancestors() -> None: ) +def test_affected_scopes_ignore_missing_paths() -> None: + assert affected_wiki_scopes(None) == ("",) + + +def test_projection_request_rejects_invalid_contract_fields() -> None: + request = _request() + + with pytest.raises(ValueError, match="requires a project_id"): + replace(request, project_id=" ") + with pytest.raises(ValueError, match="cannot be negative"): + replace(request, through_partition_position=-1) + with pytest.raises(ValueError, match="requires a projector_version"): + replace(request, projector_version=" ") + + +def test_projection_request_normalizes_and_deduplicates_scopes() -> None: + request = _request(scopes=("guides\\deep", "guides/deep", "")) + + assert request.requested_scopes == ("", "guides/deep") + + +def test_source_note_rejects_missing_metadata() -> None: + note = WikiSourceNote( + path="note.md", + title="Note", + note_type="Note", + checksum="checksum", + ) + + with pytest.raises(ValueError, match="requires a title"): + replace(note, title=" ") + with pytest.raises(ValueError, match="requires a note_type"): + replace(note, note_type=" ") + with pytest.raises(ValueError, match="requires a checksum"): + replace(note, checksum=" ") + + +def test_source_change_rejects_invalid_contract_fields() -> None: + change = WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.updated, + path="note.md", + title="Note", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ) + + with pytest.raises(ValueError, match="position must be positive"): + replace(change, partition_position=0) + with pytest.raises(ValueError, match="requires a title"): + replace(change, title=" ") + with pytest.raises(ValueError, match="timezone-aware"): + replace(change, accepted_at=ACCEPTED_AT.replace(tzinfo=None)) + with pytest.raises(ValueError, match="requires a source"): + replace(change, source=" ") + + +def test_source_change_normalizes_previous_path() -> None: + change = WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.moved, + path="new/note.md", + previous_path="old\\note.md", + title="Note", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ) + + assert change.previous_path == "old/note.md" + + +def test_reserved_document_requires_a_reserved_path_and_checksum() -> None: + with pytest.raises(ValueError, match="non-reserved path"): + _reserved("note.md", b"content") + with pytest.raises(ValueError, match="requires a checksum"): + WikiReservedDocument( + path="index.md", + checksum=" ", + content=b"content", + projector_owned=True, + ) + + +def test_snapshot_rejects_invalid_contract_fields() -> None: + snapshot = _snapshot() + + with pytest.raises(ValueError, match="requires a project_id"): + replace(snapshot, project_id=" ") + with pytest.raises(ValueError, match="requires a project_name"): + replace(snapshot, project_name=" ") + with pytest.raises(ValueError, match="cannot be negative"): + replace(snapshot, current_output_watermark=-1) + with pytest.raises(ValueError, match="timezone-aware"): + replace(snapshot, source_accepted_at=ACCEPTED_AT.replace(tzinfo=None)) + + +def test_snapshot_rejects_duplicate_note_paths_and_change_positions() -> None: + snapshot = _snapshot() + + with pytest.raises(ValueError, match="duplicate source note paths"): + replace(snapshot, notes=(snapshot.notes[0], snapshot.notes[0])) + with pytest.raises(ValueError, match="unique partition positions"): + replace(snapshot, changes=(snapshot.changes[0], snapshot.changes[0])) + + +def test_snapshot_rejects_case_folded_duplicate_reserved_paths() -> None: + lower = _reserved("guides/index.md", b"lower") + upper = _reserved("guides/Index.md", b"upper") + + with pytest.raises(ValueError, match="duplicate reserved document paths"): + replace(_snapshot(), reserved_documents=(lower, upper)) + + +def test_projection_result_reports_updating_when_output_lags_source() -> None: + result = WikiProjectionResult( + source_watermark=3, + output_watermark=2, + created=0, + updated=0, + unchanged=0, + conflicts=(), + warnings=(), + pending_materialization=(), + ) + + assert result.state == WikiProjectionState.updating + + +def test_projection_rejects_mismatched_project_and_stale_request() -> None: + with pytest.raises(ValueError, match="project_id differ"): + plan_wiki_projection(_request(), replace(_snapshot(), project_id="other")) + with pytest.raises(ValueError, match="older than"): + plan_wiki_projection(_request(position=2), _snapshot(output_watermark=3)) + + def test_projection_renders_root_and_affected_directory_indexes_and_logs() -> None: plan = plan_wiki_projection(_request(), _snapshot()) @@ -153,6 +292,28 @@ def test_projection_is_a_byte_identical_noop_at_the_same_watermark() -> None: assert replay.result.state == WikiProjectionState.current +def test_full_rebuild_records_unchanged_and_updated_reserved_documents() -> None: + request = _request(reason=WikiProjectionReason.manual_rebuild, scopes=()) + first = plan_wiki_projection(request, _snapshot()) + first_by_path = {write.path: write for write in first.writes} + unchanged = first_by_path["index.md"] + stale = _reserved("log.md", b"stale\n") + snapshot = _snapshot( + reserved_documents=( + _reserved(unchanged.path, unchanged.content), + stale, + ) + ) + + replay = plan_wiki_projection(request, snapshot) + + assert replay.unchanged_paths == ("index.md",) + assert replay.result.unchanged == 1 + assert replay.result.updated == 1 + updated_log = next(write for write in replay.writes if write.path == "log.md") + assert updated_log.expected_checksum == stale.checksum + + def test_pending_materialization_defers_all_bytes_without_advancing_output() -> None: plan = plan_wiki_projection(_request(), _snapshot(materialized=False)) @@ -302,6 +463,43 @@ def test_moved_change_requires_previous_path() -> None: ) +def test_created_and_moved_changes_render_in_the_log() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.created, + path="created.md", + title="Created", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + WikiSourceChange( + partition_position=2, + operation=WikiChangeOperation.moved, + path="moved.md", + previous_path="old.md", + title="Moved", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) + log = next(write.content.decode() for write in plan.writes if write.path == "log.md") + + assert "Created [[created|Created]]" in log + assert "Moved `old.md` to [[moved|Moved]]" in log + + def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: with pytest.raises(ValueError, match="project-relative"): WikiSourceNote( @@ -332,6 +530,7 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> "notes/code`span.md", "notes/html None: @@ -344,6 +543,27 @@ def test_wikilink_delimiters_are_rejected_at_the_contract_boundary(path: str) -> ) +@pytest.mark.parametrize("path", ("", "note.txt")) +def test_non_markdown_note_paths_are_rejected(path: str) -> None: + with pytest.raises(ValueError, match="project-relative Markdown"): + WikiSourceNote( + path=path, + title="Unsupported", + note_type="Note", + checksum="checksum", + ) + + +def test_parent_segments_are_rejected_at_the_contract_boundary() -> None: + with pytest.raises(ValueError, match="project-relative and normalized"): + WikiSourceNote( + path="notes/../outside.md", + title="Outside", + note_type="Note", + checksum="checksum", + ) + + def test_projection_order_is_deterministic_for_case_only_names() -> None: notes = ( WikiSourceNote(path="foo.md", title="same", note_type="Note", checksum="lower"), From 469f28452ac0111f215ce9bcd953ea5c7a6d94f0 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 15:58:43 -0500 Subject: [PATCH 05/19] Keep changed Wiki scopes load-bearing Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 16 +++++------ tests/indexing/test_wiki_projector.py | 31 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 5be370a3f..9b4d6973a 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -417,17 +417,15 @@ def _projection_scopes( change.previous_path for change in snapshot.changes if change.previous_path is not None ) return affected_wiki_scopes(*paths) - if request.requested_scopes: - scopes = {""} - for requested_scope in request.requested_scopes: - scope = PurePosixPath(requested_scope) - while scope != PurePosixPath("."): - scopes.add(scope.as_posix()) - scope = scope.parent - return tuple(sorted(scopes)) paths = [change.path for change in new_changes] paths.extend(change.previous_path for change in new_changes if change.previous_path is not None) - return affected_wiki_scopes(*paths) + scopes = set(affected_wiki_scopes(*paths)) + for requested_scope in request.requested_scopes: + scope = PurePosixPath(requested_scope) + while scope != PurePosixPath("."): + scopes.add(scope.as_posix()) + scope = scope.parent + return tuple(sorted(scopes)) def _render_index( diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index ca322b9f6..9b1a720d0 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -380,6 +380,37 @@ def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: assert plan.result.state == WikiProjectionState.current +def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: + changed_note = WikiSourceNote( + path="secret/note.md", + title="Secret note", + note_type="Note", + checksum="secret-checksum", + ) + changed = WikiSourceChange( + partition_position=3, + operation=WikiChangeOperation.updated, + path=changed_note.path, + title=changed_note.title, + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ) + snapshot = replace( + _snapshot(), + notes=(*_snapshot().notes, changed_note), + changes=(changed,), + ) + + plan = plan_wiki_projection(_request(scopes=("guides",)), snapshot) + + assert {write.path for write in plan.writes} >= { + "secret/index.md", + "secret/log.md", + } + assert plan.result.output_watermark == 3 + + def test_full_rebuild_covers_every_note_directory() -> None: request = _request( reason=WikiProjectionReason.import_rebuild, From 09b5af2b6ed781185512d303abf3b033b7749a87 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:06:10 -0500 Subject: [PATCH 06/19] Validate Wiki projector contract inputs Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 25 ++++++++++++++++++--- tests/indexing/test_wiki_projector.py | 24 ++++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 9b4d6973a..000c093b1 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -11,9 +11,15 @@ OKF_VERSION = "0.2" WIKI_PROFILE = "wiki/1" +WIKI_PROJECTOR_VERSION = "wiki/1.0.0" WIKI_PROJECTOR_NAME = "Basic Memory Wiki Projector" WIKI_PROJECTOR_SOURCE = "wiki_projector" RESERVED_WIKI_FILENAMES = frozenset({"index.md", "log.md"}) +WINDOWS_RESERVED_NAMES = frozenset( + {"CON", "PRN", "AUX", "NUL"} + | {f"COM{number}" for number in range(1, 10)} + | {f"LPT{number}" for number in range(1, 10)} +) class WikiProjectionReason(StrEnum): @@ -59,8 +65,8 @@ def __post_init__(self) -> None: raise ValueError("Wiki projection requires a project_id") if self.through_partition_position < 0: raise ValueError("Wiki projection position cannot be negative") - if not self.projector_version.strip(): - raise ValueError("Wiki projection requires a projector_version") + if self.projector_version != WIKI_PROJECTOR_VERSION: + raise ValueError(f"Wiki projection requires projector_version {WIKI_PROJECTOR_VERSION}") normalized_scopes = tuple( sorted({_normalize_scope(scope) for scope in self.requested_scopes}) ) @@ -596,7 +602,20 @@ def _normalize_note_path(path: str) -> str: def _normalize_scope(scope: str) -> str: - return _normalize_relative_path(scope) + normalized = _normalize_relative_path(scope) + if not normalized: + return "" + if "::" in normalized or any(character in normalized for character in "\x00\r\n[]|`<>"): + raise ValueError(f"Wiki scope contains unsupported Markdown delimiters: {scope}") + for component in PurePosixPath(normalized).parts: + if any(character in component for character in ':"?*'): + raise ValueError(f"Wiki scope contains a Windows-invalid character: {scope}") + if component.endswith((".", " ")): + raise ValueError(f"Wiki scope contains a non-portable path component: {scope}") + stem = component.split(".", 1)[0].upper() + if stem in WINDOWS_RESERVED_NAMES: + raise ValueError(f"Wiki scope contains a reserved device name: {scope}") + return normalized def _normalize_relative_path(path: str) -> str: diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 9b1a720d0..f4912fdfa 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -15,6 +15,7 @@ WikiProjectionResult, WikiProjectionSnapshot, WikiProjectionState, + WIKI_PROJECTOR_VERSION, WikiReservedDocument, WikiSourceChange, WikiSourceNote, @@ -34,7 +35,7 @@ def _request( return WikiProjectionRequest( project_id="project-88", through_partition_position=position, - projector_version="wiki/1.0.0", + projector_version=WIKI_PROJECTOR_VERSION, reason=reason, requested_scopes=scopes, ) @@ -116,8 +117,10 @@ def test_projection_request_rejects_invalid_contract_fields() -> None: replace(request, project_id=" ") with pytest.raises(ValueError, match="cannot be negative"): replace(request, through_partition_position=-1) - with pytest.raises(ValueError, match="requires a projector_version"): + with pytest.raises(ValueError, match="requires projector_version wiki/1.0.0"): replace(request, projector_version=" ") + with pytest.raises(ValueError, match="requires projector_version wiki/1.0.0"): + replace(request, projector_version="wiki/2.0.0") def test_projection_request_normalizes_and_deduplicates_scopes() -> None: @@ -126,6 +129,23 @@ def test_projection_request_normalizes_and_deduplicates_scopes() -> None: assert request.requested_scopes == ("", "guides/deep") +@pytest.mark.parametrize( + "scope", + ( + "bad|scope", + "bad::scope", + "bad\nscope", + "bad\x00scope", + "bad:scope", + "CON", + "guides/trailing.", + ), +) +def test_projection_request_rejects_nonportable_scopes(scope: str) -> None: + with pytest.raises(ValueError, match="Wiki (path|scope)"): + _request(scopes=(scope,)) + + def test_source_note_rejects_missing_metadata() -> None: note = WikiSourceNote( path="note.md", From 42c587502f110127a910357fb13a546dbb48a81f Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:11:25 -0500 Subject: [PATCH 07/19] Share portable Wiki path validation Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 18 ++++++++++++++---- tests/indexing/test_wiki_projector.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 000c093b1..b93d5a081 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -598,6 +598,7 @@ def _normalize_note_path(path: str) -> str: raise ValueError(f"Wiki note path must be project-relative Markdown: {path}") if "::" in normalized or any(character in normalized for character in "\r\n[]|`<>"): raise ValueError(f"Wiki note path contains unsupported Markdown delimiters: {path}") + _validate_portable_path_components(normalized, path_kind="note path", source=path) return normalized @@ -607,15 +608,24 @@ def _normalize_scope(scope: str) -> str: return "" if "::" in normalized or any(character in normalized for character in "\x00\r\n[]|`<>"): raise ValueError(f"Wiki scope contains unsupported Markdown delimiters: {scope}") + _validate_portable_path_components(normalized, path_kind="scope", source=scope) + return normalized + + +def _validate_portable_path_components( + normalized: str, + *, + path_kind: str, + source: str, +) -> None: for component in PurePosixPath(normalized).parts: if any(character in component for character in ':"?*'): - raise ValueError(f"Wiki scope contains a Windows-invalid character: {scope}") + raise ValueError(f"Wiki {path_kind} contains a Windows-invalid character: {source}") if component.endswith((".", " ")): - raise ValueError(f"Wiki scope contains a non-portable path component: {scope}") + raise ValueError(f"Wiki {path_kind} contains a non-portable path component: {source}") stem = component.split(".", 1)[0].upper() if stem in WINDOWS_RESERVED_NAMES: - raise ValueError(f"Wiki scope contains a reserved device name: {scope}") - return normalized + raise ValueError(f"Wiki {path_kind} contains a reserved device name: {source}") def _normalize_relative_path(path: str) -> str: diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index f4912fdfa..fa123e0ad 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -162,6 +162,24 @@ def test_source_note_rejects_missing_metadata() -> None: replace(note, checksum=" ") +@pytest.mark.parametrize( + "path", + ( + "bad?/note.md", + "NUL/note.md", + "trailing./note.md", + ), +) +def test_source_note_rejects_nonportable_path_components(path: str) -> None: + with pytest.raises(ValueError, match="Wiki note path"): + WikiSourceNote( + path=path, + title="Note", + note_type="Note", + checksum="checksum", + ) + + def test_source_change_rejects_invalid_contract_fields() -> None: change = WikiSourceChange( partition_position=1, From 5fd2ba9874f3e3ab26793b47eafcb4338d08511f Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:21:28 -0500 Subject: [PATCH 08/19] Keep bounded Wiki projections portable Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 16 +++--- tests/indexing/test_wiki_projector.py | 56 ++++++++++++++++++++- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index b93d5a081..27e24b857 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -8,6 +8,7 @@ from hashlib import sha256 import json from pathlib import PurePosixPath, PureWindowsPath +import unicodedata OKF_VERSION = "0.2" WIKI_PROFILE = "wiki/1" @@ -318,7 +319,7 @@ def plan_wiki_projection( ), ) - scopes = _projection_scopes(request, snapshot, new_changes) + scopes = _projection_scopes(request, snapshot, changes, new_changes) existing_by_path = { document.path.casefold(): document for document in snapshot.reserved_documents } @@ -413,15 +414,14 @@ def plan_wiki_projection( def _projection_scopes( request: WikiProjectionRequest, snapshot: WikiProjectionSnapshot, + changes: tuple[WikiSourceChange, ...], new_changes: tuple[WikiSourceChange, ...], ) -> tuple[str, ...]: if request.is_full_rebuild: paths = [note.path for note in snapshot.notes] paths.extend(document.path for document in snapshot.reserved_documents) - paths.extend(change.path for change in snapshot.changes) - paths.extend( - change.previous_path for change in snapshot.changes if change.previous_path is not None - ) + paths.extend(change.path for change in changes) + paths.extend(change.previous_path for change in changes if change.previous_path is not None) return affected_wiki_scopes(*paths) paths = [change.path for change in new_changes] paths.extend(change.previous_path for change in new_changes if change.previous_path is not None) @@ -619,6 +619,8 @@ def _validate_portable_path_components( source: str, ) -> None: for component in PurePosixPath(normalized).parts: + if any(unicodedata.category(character) == "Cc" for character in component): + raise ValueError(f"Wiki {path_kind} contains a control character: {source}") if any(character in component for character in ':"?*'): raise ValueError(f"Wiki {path_kind} contains a Windows-invalid character: {source}") if component.endswith((".", " ")): @@ -629,7 +631,9 @@ def _validate_portable_path_components( def _normalize_relative_path(path: str) -> str: - accepted_path = path.strip() + if path != path.strip(): + raise ValueError(f"Wiki path must not contain boundary whitespace: {path}") + accepted_path = path windows_path = PureWindowsPath(accepted_path) if accepted_path.startswith(("/", "\\")) or windows_path.drive or windows_path.is_absolute(): raise ValueError(f"Wiki path must be project-relative and normalized: {path}") diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index fa123e0ad..68a41273f 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -136,9 +136,13 @@ def test_projection_request_normalizes_and_deduplicates_scopes() -> None: "bad::scope", "bad\nscope", "bad\x00scope", + "bad\x01scope", + "bad\x7fscope", "bad:scope", "CON", "guides/trailing.", + " guides", + "guides ", ), ) def test_projection_request_rejects_nonportable_scopes(scope: str) -> None: @@ -166,12 +170,16 @@ def test_source_note_rejects_missing_metadata() -> None: "path", ( "bad?/note.md", + "bad\x01/note.md", + "bad\x7f/note.md", "NUL/note.md", "trailing./note.md", + " note.md", + "note.md ", ), ) def test_source_note_rejects_nonportable_path_components(path: str) -> None: - with pytest.raises(ValueError, match="Wiki note path"): + with pytest.raises(ValueError, match="Wiki (path|note path)"): WikiSourceNote( path=path, title="Note", @@ -507,6 +515,52 @@ def test_full_rebuild_covers_orphaned_reserved_document_scopes() -> None: assert "Deleted `orphaned/last-note.md`" in rendered["orphaned/log.md"] +def test_full_rebuild_excludes_scopes_after_requested_watermark() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.deleted, + path="included/note.md", + title="Included", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + WikiSourceChange( + partition_position=2, + operation=WikiChangeOperation.deleted, + path="future/note.md", + title="Future", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection( + _request( + position=1, + reason=WikiProjectionReason.manual_rebuild, + scopes=(), + ), + snapshot, + ) + + assert {write.path for write in plan.writes} == { + "index.md", + "log.md", + "included/index.md", + "included/log.md", + } + + def test_moved_change_requires_previous_path() -> None: with pytest.raises(ValueError, match="requires previous_path"): plan_wiki_projection( From 515a14ad8020b9b3a59d170feabe5f75373029bc Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:32:39 -0500 Subject: [PATCH 09/19] Keep Wiki snapshots bounded and portable Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 35 +++++++-- tests/indexing/test_wiki_projector.py | 85 ++++++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 27e24b857..de3f55be6 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -89,9 +89,12 @@ class WikiSourceNote: title: str note_type: str checksum: str + partition_position: int def __post_init__(self) -> None: object.__setattr__(self, "path", _normalize_note_path(self.path)) + if self.partition_position < 0: + raise ValueError("Wiki source note position cannot be negative") if not self.title.strip(): raise ValueError(f"Wiki source note {self.path} requires a title") if not self.note_type.strip(): @@ -319,15 +322,16 @@ def plan_wiki_projection( ), ) - scopes = _projection_scopes(request, snapshot, changes, new_changes) - existing_by_path = { - document.path.casefold(): document for document in snapshot.reserved_documents - } notes = tuple( note for note in snapshot.notes + if note.partition_position <= request.through_partition_position if PurePosixPath(note.path).name.lower() not in RESERVED_WIKI_FILENAMES ) + scopes = _projection_scopes(request, snapshot, notes, changes, new_changes) + existing_by_path = { + document.path.casefold(): document for document in snapshot.reserved_documents + } rendered: dict[str, bytes] = {} for scope in scopes: rendered[_reserved_path(scope, "index.md")] = _render_index( @@ -414,11 +418,12 @@ def plan_wiki_projection( def _projection_scopes( request: WikiProjectionRequest, snapshot: WikiProjectionSnapshot, + notes: tuple[WikiSourceNote, ...], changes: tuple[WikiSourceChange, ...], new_changes: tuple[WikiSourceChange, ...], ) -> tuple[str, ...]: if request.is_full_rebuild: - paths = [note.path for note in snapshot.notes] + paths = [note.path for note in notes] paths.extend(document.path for document in snapshot.reserved_documents) paths.extend(change.path for change in changes) paths.extend(change.previous_path for change in changes if change.previous_path is not None) @@ -599,6 +604,11 @@ def _normalize_note_path(path: str) -> str: if "::" in normalized or any(character in normalized for character in "\r\n[]|`<>"): raise ValueError(f"Wiki note path contains unsupported Markdown delimiters: {path}") _validate_portable_path_components(normalized, path_kind="note path", source=path) + _reject_reserved_wiki_directory_components( + PurePosixPath(normalized).parts[:-1], + path_kind="note path", + source=path, + ) return normalized @@ -609,9 +619,24 @@ def _normalize_scope(scope: str) -> str: if "::" in normalized or any(character in normalized for character in "\x00\r\n[]|`<>"): raise ValueError(f"Wiki scope contains unsupported Markdown delimiters: {scope}") _validate_portable_path_components(normalized, path_kind="scope", source=scope) + _reject_reserved_wiki_directory_components( + PurePosixPath(normalized).parts, + path_kind="scope", + source=scope, + ) return normalized +def _reject_reserved_wiki_directory_components( + components: tuple[str, ...], + *, + path_kind: str, + source: str, +) -> None: + if any(component.casefold() in RESERVED_WIKI_FILENAMES for component in components): + raise ValueError(f"Wiki {path_kind} contains a reserved Wiki directory name: {source}") + + def _validate_portable_path_components( normalized: str, *, diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 68a41273f..00f2d53c6 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -58,18 +58,21 @@ def _snapshot( title="Overview", note_type="Note", checksum="overview-checksum", + partition_position=0, ), WikiSourceNote( path="guides/setup.md", title="Setup", note_type="Guide", checksum="setup-checksum", + partition_position=3, ), WikiSourceNote( path="guides/deep/details.md", title="Details", note_type="Guide", checksum="details-checksum", + partition_position=0, ), ), changes=( @@ -156,6 +159,7 @@ def test_source_note_rejects_missing_metadata() -> None: title="Note", note_type="Note", checksum="checksum", + partition_position=0, ) with pytest.raises(ValueError, match="requires a title"): @@ -164,6 +168,8 @@ def test_source_note_rejects_missing_metadata() -> None: replace(note, note_type=" ") with pytest.raises(ValueError, match="requires a checksum"): replace(note, checksum=" ") + with pytest.raises(ValueError, match="position cannot be negative"): + replace(note, partition_position=-1) @pytest.mark.parametrize( @@ -185,9 +191,28 @@ def test_source_note_rejects_nonportable_path_components(path: str) -> None: title="Note", note_type="Note", checksum="checksum", + partition_position=0, + ) + + +@pytest.mark.parametrize("path", ("index.md/note.md", "guides/LOG.md/note.md")) +def test_source_note_rejects_reserved_wiki_directory_components(path: str) -> None: + with pytest.raises(ValueError, match="reserved Wiki directory name"): + WikiSourceNote( + path=path, + title="Note", + note_type="Note", + checksum="checksum", + partition_position=0, ) +@pytest.mark.parametrize("scope", ("index.md", "guides/LOG.md")) +def test_request_rejects_reserved_wiki_directory_scopes(scope: str) -> None: + with pytest.raises(ValueError, match="reserved Wiki directory name"): + _request(scopes=(scope,)) + + def test_source_change_rejects_invalid_contract_fields() -> None: change = WikiSourceChange( partition_position=1, @@ -432,6 +457,7 @@ def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: title="Secret note", note_type="Note", checksum="secret-checksum", + partition_position=3, ) changed = WikiSourceChange( partition_position=3, @@ -561,6 +587,43 @@ def test_full_rebuild_excludes_scopes_after_requested_watermark() -> None: } +def test_projection_excludes_notes_after_requested_watermark() -> None: + snapshot = replace( + _snapshot(output_watermark=0), + notes=( + WikiSourceNote( + path="included.md", + title="Included", + note_type="Note", + checksum="included", + partition_position=1, + ), + WikiSourceNote( + path="future/note.md", + title="Future", + note_type="Note", + checksum="future", + partition_position=2, + ), + ), + changes=(), + ) + + plan = plan_wiki_projection( + _request( + position=1, + reason=WikiProjectionReason.manual_rebuild, + scopes=(), + ), + snapshot, + ) + + assert {write.path for write in plan.writes} == {"index.md", "log.md"} + root_index = next(write.content.decode() for write in plan.writes if write.path == "index.md") + assert "[[included|Included]]" in root_index + assert "future" not in root_index.lower() + + def test_moved_change_requires_previous_path() -> None: with pytest.raises(ValueError, match="requires previous_path"): plan_wiki_projection( @@ -630,6 +693,7 @@ def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: title="Outside", note_type="Note", checksum="checksum", + partition_position=0, ) @@ -641,6 +705,7 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> title="Outside", note_type="Note", checksum="checksum", + partition_position=0, ) @@ -663,6 +728,7 @@ def test_wikilink_delimiters_are_rejected_at_the_contract_boundary(path: str) -> title="Unsupported", note_type="Note", checksum="checksum", + partition_position=0, ) @@ -674,6 +740,7 @@ def test_non_markdown_note_paths_are_rejected(path: str) -> None: title="Unsupported", note_type="Note", checksum="checksum", + partition_position=0, ) @@ -684,13 +751,26 @@ def test_parent_segments_are_rejected_at_the_contract_boundary() -> None: title="Outside", note_type="Note", checksum="checksum", + partition_position=0, ) def test_projection_order_is_deterministic_for_case_only_names() -> None: notes = ( - WikiSourceNote(path="foo.md", title="same", note_type="Note", checksum="lower"), - WikiSourceNote(path="Foo.md", title="Same", note_type="Note", checksum="upper"), + WikiSourceNote( + path="foo.md", + title="same", + note_type="Note", + checksum="lower", + partition_position=0, + ), + WikiSourceNote( + path="Foo.md", + title="Same", + note_type="Note", + checksum="upper", + partition_position=0, + ), ) snapshot = WikiProjectionSnapshot( project_id="project-88", @@ -728,6 +808,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: title=injected_title, note_type="Note", checksum="unsafe", + partition_position=1, ), ), changes=( From 279b5de75898f792b1b517743173a53c847ba7b1 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:38:53 -0500 Subject: [PATCH 10/19] Require exact Wiki projection snapshots Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 34 ++------ tests/indexing/test_wiki_projector.py | 92 +++++---------------- 2 files changed, 27 insertions(+), 99 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index de3f55be6..c3fcb02e1 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -89,12 +89,9 @@ class WikiSourceNote: title: str note_type: str checksum: str - partition_position: int def __post_init__(self) -> None: object.__setattr__(self, "path", _normalize_note_path(self.path)) - if self.partition_position < 0: - raise ValueError("Wiki source note position cannot be negative") if not self.title.strip(): raise ValueError(f"Wiki source note {self.path} requires a title") if not self.note_type.strip(): @@ -154,6 +151,7 @@ class WikiProjectionSnapshot: project_id: str project_name: str + source_partition_position: int current_output_watermark: int source_accepted_at: datetime notes: tuple[WikiSourceNote, ...] @@ -165,6 +163,8 @@ def __post_init__(self) -> None: raise ValueError("Wiki projection snapshot requires a project_id") if not self.project_name.strip(): raise ValueError("Wiki projection snapshot requires a project_name") + if self.source_partition_position < 0: + raise ValueError("Wiki snapshot source position cannot be negative") if self.current_output_watermark < 0: raise ValueError("Wiki output watermark cannot be negative") if self.source_accepted_at.tzinfo is None: @@ -254,6 +254,8 @@ def plan_wiki_projection( raise ValueError("Wiki projection request and snapshot project_id differ") if request.through_partition_position < snapshot.current_output_watermark: raise ValueError("Wiki projection request is older than the current output watermark") + if request.through_partition_position != snapshot.source_partition_position: + raise ValueError("Wiki projection requires an exact as-of source snapshot") changes = tuple( sorted( @@ -292,40 +294,14 @@ def plan_wiki_projection( ), ) - # A projector-only replay advances the ledger without rewriting its own - # generated notes. Full rebuilds and project creation remain explicit work. new_changes = tuple( change for change in changes if change.partition_position > snapshot.current_output_watermark ) - if ( - request.reason == WikiProjectionReason.accepted_note - and not new_changes - and snapshot.reserved_documents - ): - return WikiProjectionPlan( - request=request, - writes=(), - unchanged_paths=tuple( - sorted(document.path for document in snapshot.reserved_documents) - ), - result=WikiProjectionResult( - source_watermark=request.through_partition_position, - output_watermark=request.through_partition_position, - created=0, - updated=0, - unchanged=len(snapshot.reserved_documents), - conflicts=(), - warnings=(), - pending_materialization=(), - ), - ) - notes = tuple( note for note in snapshot.notes - if note.partition_position <= request.through_partition_position if PurePosixPath(note.path).name.lower() not in RESERVED_WIKI_FILENAMES ) scopes = _projection_scopes(request, snapshot, notes, changes, new_changes) diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 00f2d53c6..c59d7344a 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -50,6 +50,7 @@ def _snapshot( return WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=3, current_output_watermark=output_watermark, source_accepted_at=ACCEPTED_AT, notes=( @@ -58,21 +59,18 @@ def _snapshot( title="Overview", note_type="Note", checksum="overview-checksum", - partition_position=0, ), WikiSourceNote( path="guides/setup.md", title="Setup", note_type="Guide", checksum="setup-checksum", - partition_position=3, ), WikiSourceNote( path="guides/deep/details.md", title="Details", note_type="Guide", checksum="details-checksum", - partition_position=0, ), ), changes=( @@ -159,7 +157,6 @@ def test_source_note_rejects_missing_metadata() -> None: title="Note", note_type="Note", checksum="checksum", - partition_position=0, ) with pytest.raises(ValueError, match="requires a title"): @@ -168,8 +165,6 @@ def test_source_note_rejects_missing_metadata() -> None: replace(note, note_type=" ") with pytest.raises(ValueError, match="requires a checksum"): replace(note, checksum=" ") - with pytest.raises(ValueError, match="position cannot be negative"): - replace(note, partition_position=-1) @pytest.mark.parametrize( @@ -191,7 +186,6 @@ def test_source_note_rejects_nonportable_path_components(path: str) -> None: title="Note", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -203,7 +197,6 @@ def test_source_note_rejects_reserved_wiki_directory_components(path: str) -> No title="Note", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -268,6 +261,8 @@ def test_snapshot_rejects_invalid_contract_fields() -> None: replace(snapshot, project_id=" ") with pytest.raises(ValueError, match="requires a project_name"): replace(snapshot, project_name=" ") + with pytest.raises(ValueError, match="source position cannot be negative"): + replace(snapshot, source_partition_position=-1) with pytest.raises(ValueError, match="cannot be negative"): replace(snapshot, current_output_watermark=-1) with pytest.raises(ValueError, match="timezone-aware"): @@ -427,6 +422,7 @@ def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=4, current_output_watermark=3, source_accepted_at=ACCEPTED_AT, notes=(), @@ -446,7 +442,8 @@ def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: plan = plan_wiki_projection(_request(position=4, scopes=()), snapshot) - assert plan.writes == () + rendered = "\n".join(write.content.decode() for write in plan.writes) + assert "Updated [[index|Project 88]]" not in rendered assert plan.result.output_watermark == 4 assert plan.result.state == WikiProjectionState.current @@ -457,7 +454,6 @@ def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: title="Secret note", note_type="Note", checksum="secret-checksum", - partition_position=3, ) changed = WikiSourceChange( partition_position=3, @@ -511,6 +507,7 @@ def test_full_rebuild_covers_orphaned_reserved_document_scopes() -> None: snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=3, current_output_watermark=2, source_accepted_at=ACCEPTED_AT, notes=(), @@ -541,10 +538,11 @@ def test_full_rebuild_covers_orphaned_reserved_document_scopes() -> None: assert "Deleted `orphaned/last-note.md`" in rendered["orphaned/log.md"] -def test_full_rebuild_excludes_scopes_after_requested_watermark() -> None: +def test_projection_rejects_a_snapshot_ahead_of_the_requested_watermark() -> None: snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=2, current_output_watermark=0, source_accepted_at=ACCEPTED_AT, notes=(), @@ -570,58 +568,15 @@ def test_full_rebuild_excludes_scopes_after_requested_watermark() -> None: ), ) - plan = plan_wiki_projection( - _request( - position=1, - reason=WikiProjectionReason.manual_rebuild, - scopes=(), - ), - snapshot, - ) - - assert {write.path for write in plan.writes} == { - "index.md", - "log.md", - "included/index.md", - "included/log.md", - } - - -def test_projection_excludes_notes_after_requested_watermark() -> None: - snapshot = replace( - _snapshot(output_watermark=0), - notes=( - WikiSourceNote( - path="included.md", - title="Included", - note_type="Note", - checksum="included", - partition_position=1, - ), - WikiSourceNote( - path="future/note.md", - title="Future", - note_type="Note", - checksum="future", - partition_position=2, + with pytest.raises(ValueError, match="exact as-of source snapshot"): + plan_wiki_projection( + _request( + position=1, + reason=WikiProjectionReason.manual_rebuild, + scopes=(), ), - ), - changes=(), - ) - - plan = plan_wiki_projection( - _request( - position=1, - reason=WikiProjectionReason.manual_rebuild, - scopes=(), - ), - snapshot, - ) - - assert {write.path for write in plan.writes} == {"index.md", "log.md"} - root_index = next(write.content.decode() for write in plan.writes if write.path == "index.md") - assert "[[included|Included]]" in root_index - assert "future" not in root_index.lower() + snapshot, + ) def test_moved_change_requires_previous_path() -> None: @@ -631,6 +586,7 @@ def test_moved_change_requires_previous_path() -> None: WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=3, current_output_watermark=2, source_accepted_at=ACCEPTED_AT, notes=(), @@ -653,6 +609,7 @@ def test_created_and_moved_changes_render_in_the_log() -> None: snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=2, current_output_watermark=0, source_accepted_at=ACCEPTED_AT, notes=(), @@ -693,7 +650,6 @@ def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: title="Outside", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -705,7 +661,6 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> title="Outside", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -728,7 +683,6 @@ def test_wikilink_delimiters_are_rejected_at_the_contract_boundary(path: str) -> title="Unsupported", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -740,7 +694,6 @@ def test_non_markdown_note_paths_are_rejected(path: str) -> None: title="Unsupported", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -751,7 +704,6 @@ def test_parent_segments_are_rejected_at_the_contract_boundary() -> None: title="Outside", note_type="Note", checksum="checksum", - partition_position=0, ) @@ -762,19 +714,18 @@ def test_projection_order_is_deterministic_for_case_only_names() -> None: title="same", note_type="Note", checksum="lower", - partition_position=0, ), WikiSourceNote( path="Foo.md", title="Same", note_type="Note", checksum="upper", - partition_position=0, ), ) snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=0, current_output_watermark=0, source_accepted_at=ACCEPTED_AT, notes=notes, @@ -783,6 +734,7 @@ def test_projection_order_is_deterministic_for_case_only_names() -> None: reverse_snapshot = WikiProjectionSnapshot( project_id="project-88", project_name="Project 88", + source_partition_position=0, current_output_watermark=0, source_accepted_at=ACCEPTED_AT, notes=tuple(reversed(notes)), @@ -800,6 +752,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: snapshot = WikiProjectionSnapshot( project_id="project-88", project_name=f"Project\n{injected_title}", + source_partition_position=1, current_output_watermark=0, source_accepted_at=ACCEPTED_AT, notes=( @@ -808,7 +761,6 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: title=injected_title, note_type="Note", checksum="unsafe", - partition_position=1, ), ), changes=( From f0f1846745afe29e8e0f398142a6ee4ea753625b Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:48:04 -0500 Subject: [PATCH 11/19] Reject ambiguous Wiki projection scopes Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 9 +++++++ tests/indexing/test_wiki_projector.py | 26 +++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index c3fcb02e1..231e69e38 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -305,6 +305,15 @@ def plan_wiki_projection( if PurePosixPath(note.path).name.lower() not in RESERVED_WIKI_FILENAMES ) scopes = _projection_scopes(request, snapshot, notes, changes, new_changes) + scope_by_casefold: dict[str, str] = {} + for scope in scopes: + folded_scope = scope.casefold() + if existing_scope := scope_by_casefold.get(folded_scope): + raise ValueError( + "Wiki projection scopes must be unique when compared case-insensitively: " + f"{existing_scope}, {scope}" + ) + scope_by_casefold[folded_scope] = scope existing_by_path = { document.path.casefold(): document for document in snapshot.reserved_documents } diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index c59d7344a..66ea32bde 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -286,6 +286,32 @@ def test_snapshot_rejects_case_folded_duplicate_reserved_paths() -> None: replace(_snapshot(), reserved_documents=(lower, upper)) +def test_projection_rejects_case_folded_duplicate_scopes() -> None: + snapshot = replace( + _snapshot(), + notes=( + WikiSourceNote( + path="Foo/one.md", + title="One", + note_type="Note", + checksum="one-checksum", + ), + WikiSourceNote( + path="foo/two.md", + title="Two", + note_type="Note", + checksum="two-checksum", + ), + ), + ) + + with pytest.raises(ValueError, match="unique when compared case-insensitively"): + plan_wiki_projection( + _request(reason=WikiProjectionReason.manual_rebuild, scopes=()), + snapshot, + ) + + def test_projection_result_reports_updating_when_output_lags_source() -> None: result = WikiProjectionResult( source_watermark=3, From d277160c7d957231c8e311caa58d003d5f79f729 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 16:56:11 -0500 Subject: [PATCH 12/19] Suppress projector-only metadata rewrites Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 44 +++++++++++- tests/indexing/test_wiki_projector.py | 76 +++++++++++++++------ 2 files changed, 96 insertions(+), 24 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 231e69e38..108084b73 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -299,12 +299,28 @@ def plan_wiki_projection( for change in changes if change.partition_position > snapshot.current_output_watermark ) + projector_only_advance = ( + request.reason == WikiProjectionReason.accepted_note + and not new_changes + and any( + _is_projector_change(change) + and change.partition_position > snapshot.current_output_watermark + for change in snapshot.changes + ) + ) notes = tuple( note for note in snapshot.notes if PurePosixPath(note.path).name.lower() not in RESERVED_WIKI_FILENAMES ) - scopes = _projection_scopes(request, snapshot, notes, changes, new_changes) + scopes = _projection_scopes( + request, + snapshot, + notes, + changes, + new_changes, + repair_complete_projection=projector_only_advance, + ) scope_by_casefold: dict[str, str] = {} for scope in scopes: folded_scope = scope.casefold() @@ -367,7 +383,14 @@ def plan_wiki_projection( updated = 0 for path, content in sorted(rendered.items()): existing = existing_by_path.get(path.casefold()) - if existing is not None and existing.content == content: + if existing is not None and ( + existing.content == content + or ( + projector_only_advance + and _without_projection_metadata(existing.content) + == _without_projection_metadata(content) + ) + ): unchanged_paths.append(path) continue writes.append( @@ -406,8 +429,10 @@ def _projection_scopes( notes: tuple[WikiSourceNote, ...], changes: tuple[WikiSourceChange, ...], new_changes: tuple[WikiSourceChange, ...], + *, + repair_complete_projection: bool, ) -> tuple[str, ...]: - if request.is_full_rebuild: + if request.is_full_rebuild or repair_complete_projection: paths = [note.path for note in notes] paths.extend(document.path for document in snapshot.reserved_documents) paths.extend(change.path for change in changes) @@ -424,6 +449,19 @@ def _projection_scopes( return tuple(sorted(scopes)) +def _without_projection_metadata(content: bytes) -> bytes: + frontmatter, separator, body = content.partition(b"\n---\n") + normalized_frontmatter = b"\n".join( + b" at:" + if line.startswith(b" at: ") + else b" source_watermark:" + if line.startswith(b" source_watermark: ") + else line + for line in frontmatter.split(b"\n") + ) + return normalized_frontmatter + separator + body + + def _render_index( *, snapshot: WikiProjectionSnapshot, diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 66ea32bde..5edb14bac 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -443,37 +443,71 @@ def test_user_claimed_reserved_path_matches_case_insensitively() -> None: assert plan.result.state == WikiProjectionState.conflicted -def test_projector_generated_change_is_suppressed_from_writes_and_log() -> None: - existing = _reserved("index.md", b"existing\n") - snapshot = WikiProjectionSnapshot( - project_id="project-88", - project_name="Project 88", +def test_projector_only_advance_preserves_complete_projection_bytes() -> None: + initial_snapshot = _snapshot() + initial = plan_wiki_projection( + _request(reason=WikiProjectionReason.manual_rebuild, scopes=()), + initial_snapshot, + ) + projector_change = WikiSourceChange( + partition_position=4, + operation=WikiChangeOperation.updated, + path="index.md", + title="Project 88", + accepted_at=datetime(2026, 8, 29, 18, 31, tzinfo=timezone.utc), + materialized=True, + source="wiki_projector", + ) + snapshot = replace( + initial_snapshot, source_partition_position=4, current_output_watermark=3, - source_accepted_at=ACCEPTED_AT, - notes=(), - changes=( - WikiSourceChange( - partition_position=4, - operation=WikiChangeOperation.updated, - path="index.md", - title="Project 88", - accepted_at=ACCEPTED_AT, - materialized=True, - source="wiki_projector", - ), - ), - reserved_documents=(existing,), + source_accepted_at=projector_change.accepted_at, + changes=(*initial_snapshot.changes, projector_change), + reserved_documents=tuple(_reserved(write.path, write.content) for write in initial.writes), ) plan = plan_wiki_projection(_request(position=4, scopes=()), snapshot) - rendered = "\n".join(write.content.decode() for write in plan.writes) - assert "Updated [[index|Project 88]]" not in rendered + assert plan.writes == () + assert plan.unchanged_paths == tuple(write.path for write in initial.writes) assert plan.result.output_watermark == 4 assert plan.result.state == WikiProjectionState.current +def test_projector_only_advance_repairs_missing_projection_document() -> None: + initial_snapshot = _snapshot() + initial = plan_wiki_projection( + _request(reason=WikiProjectionReason.manual_rebuild, scopes=()), + initial_snapshot, + ) + projector_change = WikiSourceChange( + partition_position=4, + operation=WikiChangeOperation.updated, + path="index.md", + title="Project 88", + accepted_at=ACCEPTED_AT, + materialized=True, + source="wiki_projector", + ) + snapshot = replace( + initial_snapshot, + source_partition_position=4, + current_output_watermark=3, + changes=(*initial_snapshot.changes, projector_change), + reserved_documents=tuple( + _reserved(write.path, write.content) + for write in initial.writes + if write.path != "guides/deep/log.md" + ), + ) + + plan = plan_wiki_projection(_request(position=4, scopes=()), snapshot) + + assert tuple(write.path for write in plan.writes) == ("guides/deep/log.md",) + assert "Updated [[index|Project 88]]" not in plan.writes[0].content.decode() + + def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: changed_note = WikiSourceNote( path="secret/note.md", From 3bb72b6d43c1a1c281ad5e8bc077c03491738210 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:02:43 -0500 Subject: [PATCH 13/19] Keep Wiki scopes and aliases portable Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 7 +++ tests/indexing/test_wiki_projector.py | 49 +++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 108084b73..83f46121c 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -321,9 +321,15 @@ def plan_wiki_projection( new_changes, repair_complete_projection=projector_only_advance, ) + note_by_path = {note.path.casefold(): note.path for note in notes} scope_by_casefold: dict[str, str] = {} for scope in scopes: folded_scope = scope.casefold() + if existing_note_path := note_by_path.get(folded_scope): + raise ValueError( + "Wiki projection scope collides with an existing source note path: " + f"{scope}, {existing_note_path}" + ) if existing_scope := scope_by_casefold.get(folded_scope): raise ValueError( "Wiki projection scopes must be unique when compared case-insensitively: " @@ -702,6 +708,7 @@ def _escape_generated_markdown_text(value: str) -> str: { "\r": " ", "\n": " ", + "&": "&", "\\": "\", "[": "[", "]": "]", diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 5edb14bac..8e334ea48 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -312,6 +312,11 @@ def test_projection_rejects_case_folded_duplicate_scopes() -> None: ) +def test_projection_rejects_scope_that_collides_with_source_note_path() -> None: + with pytest.raises(ValueError, match="collides with an existing source note path"): + plan_wiki_projection(_request(scopes=("overview.md",)), _snapshot()) + + def test_projection_result_reports_updating_when_output_lags_source() -> None: result = WikiProjectionResult( source_watermark=3, @@ -842,3 +847,47 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: assert "\n- relates_to [[evil" not in rendered["index.md"] assert "\n- relates_to [[evil" not in rendered["log.md"] assert "[[safe-target|Bad]] - relates_to [[evil]]" in rendered["index.md"] + + +@pytest.mark.parametrize( + ("title", "escaped_title"), + ( + ("A & B", "A &amp; B"), + ("A ] B", "A &#93; B"), + ), +) +def test_projection_preserves_literal_entity_looking_titles( + title: str, + escaped_title: str, +) -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=1, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="entity-title.md", + title=title, + note_type="Note", + checksum="entity-title", + ), + ), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.created, + path="entity-title.md", + title=title, + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection(_request(position=1, scopes=()), snapshot) + rendered = {write.path: write.content.decode() for write in plan.writes} + + assert f"[[entity-title|{escaped_title}]]" in rendered["index.md"] From 58b1347ca8e3b47a2c6fe9a0c1848744e7274c92 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:09:38 -0500 Subject: [PATCH 14/19] Preserve Wiki log paths in code spans Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 7 ++-- tests/indexing/test_wiki_projector.py | 39 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 83f46121c..337ef6421 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -582,12 +582,9 @@ def _render_log_entry(change: WikiSourceChange) -> str: case WikiChangeOperation.moved: if change.previous_path is None: raise ValueError("Moved Wiki change requires previous_path") - description = ( - f"Moved `{_escape_generated_markdown_text(change.previous_path)}` to " - f"[[{path}|{title}]]" - ) + description = f"Moved `{change.previous_path}` to [[{path}|{title}]]" case WikiChangeOperation.deleted: - description = f"Deleted `{_escape_generated_markdown_text(change.path)}`" + description = f"Deleted `{change.path}`" return f"- {timestamp} — {description}" diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 8e334ea48..a93de4d4d 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -708,6 +708,45 @@ def test_created_and_moved_changes_render_in_the_log() -> None: assert "Moved `old.md` to [[moved|Moved]]" in log +def test_log_preserves_ampersands_in_code_formatted_paths() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=2, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.moved, + path="new.md", + previous_path="old&draft.md", + title="Moved", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + WikiSourceChange( + partition_position=2, + operation=WikiChangeOperation.deleted, + path="retired&archived.md", + title="Deleted", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) + log = next(write.content.decode() for write in plan.writes if write.path == "log.md") + + assert "Moved `old&draft.md` to [[new|Moved]]" in log + assert "Deleted `retired&archived.md`" in log + assert "&" not in log + + def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: with pytest.raises(ValueError, match="project-relative"): WikiSourceNote( From 2e5b940b4b7474820c8d36bec2e0c1d295f73a08 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:16:35 -0500 Subject: [PATCH 15/19] Reject noncanonical Wiki paths Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 9 +++++++-- tests/indexing/test_wiki_projector.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 337ef6421..93c3b85fa 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -689,11 +689,16 @@ def _normalize_relative_path(path: str) -> str: if accepted_path.startswith(("/", "\\")) or windows_path.drive or windows_path.is_absolute(): raise ValueError(f"Wiki path must be project-relative and normalized: {path}") candidate = accepted_path.replace("\\", "/") - candidate = candidate.strip("/") if not candidate: return "" + if candidate.endswith("/"): + raise ValueError(f"Wiki path must be project-relative and normalized: {path}") parsed = PurePosixPath(candidate) - if parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts): + if ( + parsed.is_absolute() + or any(part in {"", ".", ".."} for part in parsed.parts) + or parsed.as_posix() != candidate + ): raise ValueError(f"Wiki path must be project-relative and normalized: {path}") return parsed.as_posix() diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index a93de4d4d..3dc792d25 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -768,6 +768,17 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> ) +@pytest.mark.parametrize("path", ("notes//foo.md", "notes/./foo.md", "note.md/")) +def test_noncanonical_paths_are_rejected_at_the_contract_boundary(path: str) -> None: + with pytest.raises(ValueError, match="project-relative"): + WikiSourceNote( + path=path, + title="Noncanonical", + note_type="Note", + checksum="checksum", + ) + + @pytest.mark.parametrize( "path", ( From 5cb68c3879c7d76f8989a3d50945c1a45c1fe868 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:22:33 -0500 Subject: [PATCH 16/19] Normalize Wiki projection path keys Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 27 ++++++++++++--------- tests/indexing/test_wiki_projector.py | 23 +++++++++++++++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 93c3b85fa..8ac51ddd4 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -321,23 +321,23 @@ def plan_wiki_projection( new_changes, repair_complete_projection=projector_only_advance, ) - note_by_path = {note.path.casefold(): note.path for note in notes} - scope_by_casefold: dict[str, str] = {} + note_by_path = {_portable_path_key(note.path): note.path for note in notes} + scope_by_portable_path: dict[str, str] = {} for scope in scopes: - folded_scope = scope.casefold() - if existing_note_path := note_by_path.get(folded_scope): + portable_scope = _portable_path_key(scope) + if existing_note_path := note_by_path.get(portable_scope): raise ValueError( "Wiki projection scope collides with an existing source note path: " f"{scope}, {existing_note_path}" ) - if existing_scope := scope_by_casefold.get(folded_scope): + if existing_scope := scope_by_portable_path.get(portable_scope): raise ValueError( - "Wiki projection scopes must be unique when compared case-insensitively: " + "Wiki projection scopes must be unique when compared as portable paths: " f"{existing_scope}, {scope}" ) - scope_by_casefold[folded_scope] = scope + scope_by_portable_path[portable_scope] = scope existing_by_path = { - document.path.casefold(): document for document in snapshot.reserved_documents + _portable_path_key(document.path): document for document in snapshot.reserved_documents } rendered: dict[str, bytes] = {} for scope in scopes: @@ -360,7 +360,7 @@ def plan_wiki_projection( reason="reserved path is not owned by the Wiki Projector", ) for path in sorted(rendered) - if (existing := existing_by_path.get(path.casefold())) is not None + if (existing := existing_by_path.get(_portable_path_key(path))) is not None and not existing.projector_owned ) if conflicts: @@ -388,7 +388,7 @@ def plan_wiki_projection( created = 0 updated = 0 for path, content in sorted(rendered.items()): - existing = existing_by_path.get(path.casefold()) + existing = existing_by_path.get(_portable_path_key(path)) if existing is not None and ( existing.content == content or ( @@ -731,11 +731,16 @@ def _require_unique_paths( ) -> None: paths = [getattr(value, "path") for value in values] if not case_sensitive: - paths = [path.casefold() for path in paths] + paths = [_portable_path_key(path) for path in paths] if len(paths) != len(set(paths)): raise ValueError(f"Wiki projection snapshot has duplicate {label} paths") +def _portable_path_key(path: str) -> str: + """Compare paths the way normalization-insensitive filesystems do.""" + return unicodedata.normalize("NFC", path).casefold() + + def _reserved_path(scope: str, filename: str) -> str: return f"{scope}/{filename}" if scope else filename diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 3dc792d25..b05e6dca1 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -286,18 +286,25 @@ def test_snapshot_rejects_case_folded_duplicate_reserved_paths() -> None: replace(_snapshot(), reserved_documents=(lower, upper)) -def test_projection_rejects_case_folded_duplicate_scopes() -> None: +@pytest.mark.parametrize( + ("first_scope", "second_scope"), + (("Foo", "foo"), ("caf\u00e9", "cafe\u0301")), +) +def test_projection_rejects_nonportable_duplicate_scopes( + first_scope: str, + second_scope: str, +) -> None: snapshot = replace( _snapshot(), notes=( WikiSourceNote( - path="Foo/one.md", + path=f"{first_scope}/one.md", title="One", note_type="Note", checksum="one-checksum", ), WikiSourceNote( - path="foo/two.md", + path=f"{second_scope}/two.md", title="Two", note_type="Note", checksum="two-checksum", @@ -305,13 +312,21 @@ def test_projection_rejects_case_folded_duplicate_scopes() -> None: ), ) - with pytest.raises(ValueError, match="unique when compared case-insensitively"): + with pytest.raises(ValueError, match="unique when compared as portable paths"): plan_wiki_projection( _request(reason=WikiProjectionReason.manual_rebuild, scopes=()), snapshot, ) +def test_snapshot_rejects_unicode_normalized_duplicate_reserved_paths() -> None: + composed = _reserved("caf\u00e9/index.md", b"composed") + decomposed = _reserved("cafe\u0301/index.md", b"decomposed") + + with pytest.raises(ValueError, match="duplicate reserved document paths"): + replace(_snapshot(), reserved_documents=(composed, decomposed)) + + def test_projection_rejects_scope_that_collides_with_source_note_path() -> None: with pytest.raises(ValueError, match="collides with an existing source note path"): plan_wiki_projection(_request(scopes=("overview.md",)), _snapshot()) From 1a8f30ee605fcbc67be1ba1eaa8599c6fe9edaaf Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:32:39 -0500 Subject: [PATCH 17/19] Emit canonical Wiki note identities Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 40 +++++++------ tests/indexing/test_wiki_projector.py | 62 ++++++++++++++++++++- 2 files changed, 83 insertions(+), 19 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 8ac51ddd4..6b6e3b6e0 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -86,12 +86,17 @@ class WikiSourceNote: """One accepted, materialized note visible to a projector snapshot.""" path: str + permalink: str title: str note_type: str checksum: str def __post_init__(self) -> None: object.__setattr__(self, "path", _normalize_note_path(self.path)) + if not self.permalink.strip() or self.permalink != self.permalink.strip(): + raise ValueError(f"Wiki source note {self.path} requires a canonical permalink") + if "::" in self.permalink or any(character in self.permalink for character in "\r\n[]|`<>"): + raise ValueError(f"Wiki source note {self.path} has an unsafe canonical permalink") if not self.title.strip(): raise ValueError(f"Wiki source note {self.path} requires a title") if not self.note_type.strip(): @@ -321,14 +326,14 @@ def plan_wiki_projection( new_changes, repair_complete_projection=projector_only_advance, ) - note_by_path = {_portable_path_key(note.path): note.path for note in notes} + note_by_path = {_portable_path_key(note.path): note for note in notes} scope_by_portable_path: dict[str, str] = {} for scope in scopes: portable_scope = _portable_path_key(scope) - if existing_note_path := note_by_path.get(portable_scope): + if existing_note := note_by_path.get(portable_scope): raise ValueError( "Wiki projection scope collides with an existing source note path: " - f"{scope}, {existing_note_path}" + f"{scope}, {existing_note.path}" ) if existing_scope := scope_by_portable_path.get(portable_scope): raise ValueError( @@ -349,6 +354,7 @@ def plan_wiki_projection( ) rendered[_reserved_path(scope, "log.md")] = _render_log( snapshot=snapshot, + notes=notes, changes=changes, scope=scope, source_watermark=request.through_partition_position, @@ -506,9 +512,7 @@ def _render_index( if direct_notes: body.extend(["## Notes", ""]) body.extend( - "- " - f"[[{_without_markdown_suffix(note.path)}|" - f"{_escape_generated_markdown_text(note.title)}]]" + f"- [[{note.permalink}|{_escape_generated_markdown_text(note.title)}]]" for note in direct_notes ) body.append("") @@ -527,6 +531,7 @@ def _render_index( def _render_log( *, snapshot: WikiProjectionSnapshot, + notes: tuple[WikiSourceNote, ...], changes: tuple[WikiSourceChange, ...], scope: str, source_watermark: int, @@ -555,8 +560,15 @@ def _render_log( else f"{_display_name(PurePosixPath(scope).name)} log" ) body: list[str] = [f"# {_escape_generated_markdown_text(title)}", ""] + note_permalink_by_path = {_portable_path_key(note.path): note.permalink for note in notes} if relevant: - body.extend(_render_log_entry(change) for change in relevant) + body.extend( + _render_log_entry( + change, + note_permalink_by_path.get(_portable_path_key(change.path)), + ) + for change in relevant + ) body.append("") else: body.extend(["No accepted materialized changes have been recorded yet.", ""]) @@ -570,19 +582,19 @@ def _render_log( ) -def _render_log_entry(change: WikiSourceChange) -> str: +def _render_log_entry(change: WikiSourceChange, permalink: str | None) -> str: timestamp = _isoformat_utc(change.accepted_at) - path = _without_markdown_suffix(change.path) title = _escape_generated_markdown_text(change.title) + current_note = f"[[{permalink}|{title}]]" if permalink is not None else f"`{change.path}`" match change.operation: case WikiChangeOperation.created: - description = f"Created [[{path}|{title}]]" + description = f"Created {current_note}" case WikiChangeOperation.updated: - description = f"Updated [[{path}|{title}]]" + description = f"Updated {current_note}" case WikiChangeOperation.moved: if change.previous_path is None: raise ValueError("Moved Wiki change requires previous_path") - description = f"Moved `{change.previous_path}` to [[{path}|{title}]]" + description = f"Moved `{change.previous_path}` to {current_note}" case WikiChangeOperation.deleted: description = f"Deleted `{change.path}`" return f"- {timestamp} — {description}" @@ -769,9 +781,5 @@ def _display_name(value: str) -> str: return value.replace("-", " ").replace("_", " ").strip().title() -def _without_markdown_suffix(path: str) -> str: - return path[:-3] if path.lower().endswith(".md") else path - - def _isoformat_utc(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index b05e6dca1..dde211852 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -56,18 +56,21 @@ def _snapshot( notes=( WikiSourceNote( path="overview.md", + permalink="overview", title="Overview", note_type="Note", checksum="overview-checksum", ), WikiSourceNote( path="guides/setup.md", + permalink="guides/setup", title="Setup", note_type="Guide", checksum="setup-checksum", ), WikiSourceNote( path="guides/deep/details.md", + permalink="guides/deep/details", title="Details", note_type="Guide", checksum="details-checksum", @@ -154,6 +157,7 @@ def test_projection_request_rejects_nonportable_scopes(scope: str) -> None: def test_source_note_rejects_missing_metadata() -> None: note = WikiSourceNote( path="note.md", + permalink="note", title="Note", note_type="Note", checksum="checksum", @@ -165,6 +169,10 @@ def test_source_note_rejects_missing_metadata() -> None: replace(note, note_type=" ") with pytest.raises(ValueError, match="requires a checksum"): replace(note, checksum=" ") + with pytest.raises(ValueError, match="requires a canonical permalink"): + replace(note, permalink=" ") + with pytest.raises(ValueError, match="unsafe canonical permalink"): + replace(note, permalink="bad|target") @pytest.mark.parametrize( @@ -183,6 +191,7 @@ def test_source_note_rejects_nonportable_path_components(path: str) -> None: with pytest.raises(ValueError, match="Wiki (path|note path)"): WikiSourceNote( path=path, + permalink="note", title="Note", note_type="Note", checksum="checksum", @@ -194,6 +203,7 @@ def test_source_note_rejects_reserved_wiki_directory_components(path: str) -> No with pytest.raises(ValueError, match="reserved Wiki directory name"): WikiSourceNote( path=path, + permalink="note", title="Note", note_type="Note", checksum="checksum", @@ -299,12 +309,14 @@ def test_projection_rejects_nonportable_duplicate_scopes( notes=( WikiSourceNote( path=f"{first_scope}/one.md", + permalink=f"{first_scope}/one", title="One", note_type="Note", checksum="one-checksum", ), WikiSourceNote( path=f"{second_scope}/two.md", + permalink=f"{second_scope}/two", title="Two", note_type="Note", checksum="two-checksum", @@ -531,6 +543,7 @@ def test_projector_only_advance_repairs_missing_projection_document() -> None: def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: changed_note = WikiSourceNote( path="secret/note.md", + permalink="secret/note", title="Secret note", note_type="Note", checksum="secret-checksum", @@ -719,8 +732,8 @@ def test_created_and_moved_changes_render_in_the_log() -> None: plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) log = next(write.content.decode() for write in plan.writes if write.path == "log.md") - assert "Created [[created|Created]]" in log - assert "Moved `old.md` to [[moved|Moved]]" in log + assert "Created `created.md`" in log + assert "Moved `old.md` to `moved.md`" in log def test_log_preserves_ampersands_in_code_formatted_paths() -> None: @@ -757,7 +770,7 @@ def test_log_preserves_ampersands_in_code_formatted_paths() -> None: plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) log = next(write.content.decode() for write in plan.writes if write.path == "log.md") - assert "Moved `old&draft.md` to [[new|Moved]]" in log + assert "Moved `old&draft.md` to `new.md`" in log assert "Deleted `retired&archived.md`" in log assert "&" not in log @@ -766,6 +779,7 @@ def test_absolute_paths_are_rejected_at_the_contract_boundary() -> None: with pytest.raises(ValueError, match="project-relative"): WikiSourceNote( path="/outside.md", + permalink="outside", title="Outside", note_type="Note", checksum="checksum", @@ -777,6 +791,7 @@ def test_windows_drive_paths_are_rejected_at_the_contract_boundary(path: str) -> with pytest.raises(ValueError, match="project-relative"): WikiSourceNote( path=path, + permalink="outside", title="Outside", note_type="Note", checksum="checksum", @@ -788,6 +803,7 @@ def test_noncanonical_paths_are_rejected_at_the_contract_boundary(path: str) -> with pytest.raises(ValueError, match="project-relative"): WikiSourceNote( path=path, + permalink="noncanonical", title="Noncanonical", note_type="Note", checksum="checksum", @@ -810,6 +826,7 @@ def test_wikilink_delimiters_are_rejected_at_the_contract_boundary(path: str) -> with pytest.raises(ValueError, match="unsupported Markdown delimiters"): WikiSourceNote( path=path, + permalink="unsupported", title="Unsupported", note_type="Note", checksum="checksum", @@ -821,6 +838,7 @@ def test_non_markdown_note_paths_are_rejected(path: str) -> None: with pytest.raises(ValueError, match="project-relative Markdown"): WikiSourceNote( path=path, + permalink="unsupported", title="Unsupported", note_type="Note", checksum="checksum", @@ -831,6 +849,7 @@ def test_parent_segments_are_rejected_at_the_contract_boundary() -> None: with pytest.raises(ValueError, match="project-relative and normalized"): WikiSourceNote( path="notes/../outside.md", + permalink="outside", title="Outside", note_type="Note", checksum="checksum", @@ -841,12 +860,14 @@ def test_projection_order_is_deterministic_for_case_only_names() -> None: notes = ( WikiSourceNote( path="foo.md", + permalink="foo", title="same", note_type="Note", checksum="lower", ), WikiSourceNote( path="Foo.md", + permalink="foo-1", title="Same", note_type="Note", checksum="upper", @@ -877,6 +898,39 @@ def test_projection_order_is_deterministic_for_case_only_names() -> None: assert first.writes == second.writes +def test_projection_links_notes_by_their_canonical_permalinks() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=0, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="foo bar.md", + permalink="foo-bar", + title="Spaced", + note_type="Note", + checksum="spaced", + ), + WikiSourceNote( + path="foo-bar.md", + permalink="foo-bar-1", + title="Hyphenated", + note_type="Note", + checksum="hyphenated", + ), + ), + changes=(), + ) + + plan = plan_wiki_projection(_request(position=0, scopes=()), snapshot) + index = next(write.content.decode() for write in plan.writes if write.path == "index.md") + + assert "[[foo-bar|Spaced]]" in index + assert "[[foo-bar-1|Hyphenated]]" in index + + def test_projection_escapes_dynamic_markdown_structure() -> None: injected_title = "Bad]]\n- relates_to [[evil" snapshot = WikiProjectionSnapshot( @@ -888,6 +942,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: notes=( WikiSourceNote( path="safe-target.md", + permalink="safe-target", title=injected_title, note_type="Note", checksum="unsafe", @@ -934,6 +989,7 @@ def test_projection_preserves_literal_entity_looking_titles( notes=( WikiSourceNote( path="entity-title.md", + permalink="entity-title", title=title, note_type="Note", checksum="entity-title", From 9cce74735497e1e08032f33b80ad8d579f57aa63 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:40:47 -0500 Subject: [PATCH 18/19] Preserve canonical Wiki log identities Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 46 +++++++---- tests/indexing/test_wiki_projector.py | 86 ++++++++++++++++++++- 2 files changed, 113 insertions(+), 19 deletions(-) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index 6b6e3b6e0..c22ba6cdb 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -93,10 +93,7 @@ class WikiSourceNote: def __post_init__(self) -> None: object.__setattr__(self, "path", _normalize_note_path(self.path)) - if not self.permalink.strip() or self.permalink != self.permalink.strip(): - raise ValueError(f"Wiki source note {self.path} requires a canonical permalink") - if "::" in self.permalink or any(character in self.permalink for character in "\r\n[]|`<>"): - raise ValueError(f"Wiki source note {self.path} has an unsafe canonical permalink") + _validate_canonical_permalink(self.permalink, label=f"Wiki source note {self.path}") if not self.title.strip(): raise ValueError(f"Wiki source note {self.path} requires a title") if not self.note_type.strip(): @@ -112,6 +109,7 @@ class WikiSourceChange: partition_position: int operation: WikiChangeOperation path: str + permalink: str title: str accepted_at: datetime materialized: bool @@ -122,6 +120,7 @@ def __post_init__(self) -> None: if self.partition_position <= 0: raise ValueError("Wiki source change position must be positive") object.__setattr__(self, "path", _normalize_note_path(self.path)) + _validate_canonical_permalink(self.permalink, label=f"Wiki source change {self.path}") if self.previous_path is not None: object.__setattr__(self, "previous_path", _normalize_note_path(self.previous_path)) if not self.title.strip(): @@ -326,6 +325,23 @@ def plan_wiki_projection( new_changes, repair_complete_projection=projector_only_advance, ) + all_projection_paths: list[str | None] = [note.path for note in notes] + all_projection_paths.extend(document.path for document in snapshot.reserved_documents) + all_projection_paths.extend(change.path for change in changes) + all_projection_paths.extend( + change.previous_path for change in changes if change.previous_path is not None + ) + reserved_permalink_keys = { + _portable_path_key(_reserved_path(scope, filename).removesuffix(".md")) + for scope in affected_wiki_scopes(*all_projection_paths) + for filename in RESERVED_WIKI_FILENAMES + } + for note in notes: + if _portable_path_key(note.permalink) in reserved_permalink_keys: + raise ValueError( + "Wiki source note permalink collides with a generated document identity: " + f"{note.permalink}" + ) note_by_path = {_portable_path_key(note.path): note for note in notes} scope_by_portable_path: dict[str, str] = {} for scope in scopes: @@ -354,7 +370,6 @@ def plan_wiki_projection( ) rendered[_reserved_path(scope, "log.md")] = _render_log( snapshot=snapshot, - notes=notes, changes=changes, scope=scope, source_watermark=request.through_partition_position, @@ -531,7 +546,6 @@ def _render_index( def _render_log( *, snapshot: WikiProjectionSnapshot, - notes: tuple[WikiSourceNote, ...], changes: tuple[WikiSourceChange, ...], scope: str, source_watermark: int, @@ -560,15 +574,8 @@ def _render_log( else f"{_display_name(PurePosixPath(scope).name)} log" ) body: list[str] = [f"# {_escape_generated_markdown_text(title)}", ""] - note_permalink_by_path = {_portable_path_key(note.path): note.permalink for note in notes} if relevant: - body.extend( - _render_log_entry( - change, - note_permalink_by_path.get(_portable_path_key(change.path)), - ) - for change in relevant - ) + body.extend(_render_log_entry(change) for change in relevant) body.append("") else: body.extend(["No accepted materialized changes have been recorded yet.", ""]) @@ -582,10 +589,10 @@ def _render_log( ) -def _render_log_entry(change: WikiSourceChange, permalink: str | None) -> str: +def _render_log_entry(change: WikiSourceChange) -> str: timestamp = _isoformat_utc(change.accepted_at) title = _escape_generated_markdown_text(change.title) - current_note = f"[[{permalink}|{title}]]" if permalink is not None else f"`{change.path}`" + current_note = f"[[{change.permalink}|{title}]]" match change.operation: case WikiChangeOperation.created: description = f"Created {current_note}" @@ -650,6 +657,13 @@ def _normalize_note_path(path: str) -> str: return normalized +def _validate_canonical_permalink(permalink: str, *, label: str) -> None: + if not permalink.strip() or permalink != permalink.strip(): + raise ValueError(f"{label} requires a canonical permalink") + if "::" in permalink or any(character in permalink for character in "\r\n[]|`<>"): + raise ValueError(f"{label} has an unsafe canonical permalink") + + def _normalize_scope(scope: str) -> str: normalized = _normalize_relative_path(scope) if not normalized: diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index dde211852..6ec1faa61 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -81,6 +81,7 @@ def _snapshot( partition_position=3, operation=WikiChangeOperation.updated, path="guides/setup.md", + permalink="guides/setup", title="Setup", accepted_at=ACCEPTED_AT, materialized=materialized, @@ -221,6 +222,7 @@ def test_source_change_rejects_invalid_contract_fields() -> None: partition_position=1, operation=WikiChangeOperation.updated, path="note.md", + permalink="note", title="Note", accepted_at=ACCEPTED_AT, materialized=True, @@ -231,6 +233,10 @@ def test_source_change_rejects_invalid_contract_fields() -> None: replace(change, partition_position=0) with pytest.raises(ValueError, match="requires a title"): replace(change, title=" ") + with pytest.raises(ValueError, match="requires a canonical permalink"): + replace(change, permalink=" ") + with pytest.raises(ValueError, match="unsafe canonical permalink"): + replace(change, permalink="bad|target") with pytest.raises(ValueError, match="timezone-aware"): replace(change, accepted_at=ACCEPTED_AT.replace(tzinfo=None)) with pytest.raises(ValueError, match="requires a source"): @@ -242,6 +248,7 @@ def test_source_change_normalizes_previous_path() -> None: partition_position=1, operation=WikiChangeOperation.moved, path="new/note.md", + permalink="new/note", previous_path="old\\note.md", title="Note", accepted_at=ACCEPTED_AT, @@ -485,6 +492,7 @@ def test_projector_only_advance_preserves_complete_projection_bytes() -> None: partition_position=4, operation=WikiChangeOperation.updated, path="index.md", + permalink="index", title="Project 88", accepted_at=datetime(2026, 8, 29, 18, 31, tzinfo=timezone.utc), materialized=True, @@ -517,6 +525,7 @@ def test_projector_only_advance_repairs_missing_projection_document() -> None: partition_position=4, operation=WikiChangeOperation.updated, path="index.md", + permalink="index", title="Project 88", accepted_at=ACCEPTED_AT, materialized=True, @@ -552,6 +561,7 @@ def test_requested_scopes_cannot_omit_a_changed_note_scope() -> None: partition_position=3, operation=WikiChangeOperation.updated, path=changed_note.path, + permalink=changed_note.permalink, title=changed_note.title, accepted_at=ACCEPTED_AT, materialized=True, @@ -609,6 +619,7 @@ def test_full_rebuild_covers_orphaned_reserved_document_scopes() -> None: partition_position=3, operation=WikiChangeOperation.deleted, path="orphaned/last-note.md", + permalink="orphaned/last-note", title="Last note", accepted_at=ACCEPTED_AT, materialized=True, @@ -644,6 +655,7 @@ def test_projection_rejects_a_snapshot_ahead_of_the_requested_watermark() -> Non partition_position=1, operation=WikiChangeOperation.deleted, path="included/note.md", + permalink="included/note", title="Included", accepted_at=ACCEPTED_AT, materialized=True, @@ -653,6 +665,7 @@ def test_projection_rejects_a_snapshot_ahead_of_the_requested_watermark() -> Non partition_position=2, operation=WikiChangeOperation.deleted, path="future/note.md", + permalink="future/note", title="Future", accepted_at=ACCEPTED_AT, materialized=True, @@ -688,6 +701,7 @@ def test_moved_change_requires_previous_path() -> None: partition_position=3, operation=WikiChangeOperation.moved, path="guides/new.md", + permalink="guides/new", title="Moved", accepted_at=ACCEPTED_AT, materialized=True, @@ -711,6 +725,7 @@ def test_created_and_moved_changes_render_in_the_log() -> None: partition_position=1, operation=WikiChangeOperation.created, path="created.md", + permalink="created", title="Created", accepted_at=ACCEPTED_AT, materialized=True, @@ -720,6 +735,7 @@ def test_created_and_moved_changes_render_in_the_log() -> None: partition_position=2, operation=WikiChangeOperation.moved, path="moved.md", + permalink="moved", previous_path="old.md", title="Moved", accepted_at=ACCEPTED_AT, @@ -732,8 +748,8 @@ def test_created_and_moved_changes_render_in_the_log() -> None: plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) log = next(write.content.decode() for write in plan.writes if write.path == "log.md") - assert "Created `created.md`" in log - assert "Moved `old.md` to `moved.md`" in log + assert "Created [[created|Created]]" in log + assert "Moved `old.md` to [[moved|Moved]]" in log def test_log_preserves_ampersands_in_code_formatted_paths() -> None: @@ -749,6 +765,7 @@ def test_log_preserves_ampersands_in_code_formatted_paths() -> None: partition_position=1, operation=WikiChangeOperation.moved, path="new.md", + permalink="new", previous_path="old&draft.md", title="Moved", accepted_at=ACCEPTED_AT, @@ -759,6 +776,7 @@ def test_log_preserves_ampersands_in_code_formatted_paths() -> None: partition_position=2, operation=WikiChangeOperation.deleted, path="retired&archived.md", + permalink="retired-archived", title="Deleted", accepted_at=ACCEPTED_AT, materialized=True, @@ -770,7 +788,7 @@ def test_log_preserves_ampersands_in_code_formatted_paths() -> None: plan = plan_wiki_projection(_request(position=2, scopes=()), snapshot) log = next(write.content.decode() for write in plan.writes if write.path == "log.md") - assert "Moved `old&draft.md` to `new.md`" in log + assert "Moved `old&draft.md` to [[new|Moved]]" in log assert "Deleted `retired&archived.md`" in log assert "&" not in log @@ -931,6 +949,66 @@ def test_projection_links_notes_by_their_canonical_permalinks() -> None: assert "[[foo-bar-1|Hyphenated]]" in index +def test_projection_logs_preserve_each_changes_historical_permalink() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=1, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="same.md", + permalink="new-note", + title="New note", + note_type="Note", + checksum="new-note", + ), + ), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.created, + path="same.md", + permalink="old-note", + title="Old note", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + plan = plan_wiki_projection(_request(position=1, scopes=()), snapshot) + log = next(write.content.decode() for write in plan.writes if write.path == "log.md") + + assert "Created [[old-note|Old note]]" in log + assert "[[new-note|Old note]]" not in log + + +def test_projection_rejects_source_permalink_reserved_for_generated_index() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=0, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=( + WikiSourceNote( + path="guides/topic.md", + permalink="guides/index", + title="Topic", + note_type="Note", + checksum="topic", + ), + ), + changes=(), + ) + + with pytest.raises(ValueError, match="generated document identity"): + plan_wiki_projection(_request(position=0, scopes=()), snapshot) + + def test_projection_escapes_dynamic_markdown_structure() -> None: injected_title = "Bad]]\n- relates_to [[evil" snapshot = WikiProjectionSnapshot( @@ -953,6 +1031,7 @@ def test_projection_escapes_dynamic_markdown_structure() -> None: partition_position=1, operation=WikiChangeOperation.updated, path="safe-target.md", + permalink="safe-target", title=injected_title, accepted_at=ACCEPTED_AT, materialized=True, @@ -1000,6 +1079,7 @@ def test_projection_preserves_literal_entity_looking_titles( partition_position=1, operation=WikiChangeOperation.created, path="entity-title.md", + permalink="entity-title", title=title, accepted_at=ACCEPTED_AT, materialized=True, From 05b4801fb29dc68ae4542437361755aeed7852f5 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 30 Aug 2026 17:47:13 -0500 Subject: [PATCH 19/19] Reject reserved historical Wiki identities Signed-off-by: phernandez --- src/basic_memory/indexing/wiki_projector.py | 9 ++++++ tests/indexing/test_wiki_projector.py | 36 +++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/basic_memory/indexing/wiki_projector.py b/src/basic_memory/indexing/wiki_projector.py index c22ba6cdb..267911210 100644 --- a/src/basic_memory/indexing/wiki_projector.py +++ b/src/basic_memory/indexing/wiki_projector.py @@ -342,6 +342,15 @@ def plan_wiki_projection( "Wiki source note permalink collides with a generated document identity: " f"{note.permalink}" ) + for change in changes: + if ( + change.operation is not WikiChangeOperation.deleted + and _portable_path_key(change.permalink) in reserved_permalink_keys + ): + raise ValueError( + "Wiki source change permalink collides with a generated document identity: " + f"{change.permalink}" + ) note_by_path = {_portable_path_key(note.path): note for note in notes} scope_by_portable_path: dict[str, str] = {} for scope in scopes: diff --git a/tests/indexing/test_wiki_projector.py b/tests/indexing/test_wiki_projector.py index 6ec1faa61..3c23419be 100644 --- a/tests/indexing/test_wiki_projector.py +++ b/tests/indexing/test_wiki_projector.py @@ -1009,6 +1009,42 @@ def test_projection_rejects_source_permalink_reserved_for_generated_index() -> N plan_wiki_projection(_request(position=0, scopes=()), snapshot) +def test_projection_rejects_historical_permalink_reserved_for_generated_index() -> None: + snapshot = WikiProjectionSnapshot( + project_id="project-88", + project_name="Project 88", + source_partition_position=2, + current_output_watermark=0, + source_accepted_at=ACCEPTED_AT, + notes=(), + changes=( + WikiSourceChange( + partition_position=1, + operation=WikiChangeOperation.created, + path="guides/topic.md", + permalink="guides/index", + title="Deleted topic", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + WikiSourceChange( + partition_position=2, + operation=WikiChangeOperation.deleted, + path="guides/topic.md", + permalink="guides/index", + title="Deleted topic", + accepted_at=ACCEPTED_AT, + materialized=True, + source="web", + ), + ), + ) + + with pytest.raises(ValueError, match="generated document identity"): + plan_wiki_projection(_request(position=2, scopes=()), snapshot) + + def test_projection_escapes_dynamic_markdown_structure() -> None: injected_title = "Bad]]\n- relates_to [[evil" snapshot = WikiProjectionSnapshot(