From ed41fc32d9a69526cdfb053105a844ef8ec34749 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Wed, 2 Sep 2026 21:32:49 +0200 Subject: [PATCH 1/2] fix(generate): prevent sample cache churn --- docs/quickstart.md | 6 +++ src/wlens/render/markdown.py | 27 +++++++++++- src/wlens/render/sample_cache.py | 62 +++++++++++++++++++------- tests/test_render_markdown.py | 76 +++++++++++++++++++++++++++++++- 4 files changed, 153 insertions(+), 18 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 96e5584..d29b1a7 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -139,6 +139,12 @@ wlens/schema/ If your executor is configured and reachable, each file includes five sample rows. Offline / no credentials? Pass `--skip-samples`. +Sample rows are pinned in `wlens/.cache/samples/` to keep generated diffs +stable. `wlens generate` reuses them while the table's column-name hash is +unchanged (column order and data-type metadata do not count). PII flags and +obfuscation rules are checked separately so newly-sensitive values cannot leak. +Use `--refresh-samples` only when you intentionally want new samples. + ## 6. Ask your AI agent something Open Claude Code, Cursor, Continue, or Codex and ask a question. The diff --git a/src/wlens/render/markdown.py b/src/wlens/render/markdown.py index 27666d3..61b7575 100644 --- a/src/wlens/render/markdown.py +++ b/src/wlens/render/markdown.py @@ -57,6 +57,11 @@ def render_and_write_all( output_dir = config.output_dir output_dir.mkdir(parents=True, exist_ok=True) + # A dbt manifest can expose the same physical relation as both a model and + # a source. Both entities otherwise share a markdown path and sample-cache + # path, causing them to invalidate and overwrite each other on every run. + entities = _deduplicate_entities(entities) + inline_map = _inline_map(custom_entities) sample_size = config.output.sample_size if config.output.include_sample_rows else 0 obfuscate_value = functools.partial( @@ -287,7 +292,27 @@ def _write_file(path: Path, new_body: str) -> None: logger.info(f" wrote {path.name}") -# ─── Inlining custom entities ─────────────────────────────────────────────── +# ─── Duplicate relations + inlining custom entities ──────────────────────── + + +def _deduplicate_entities(entities: list[Entity]) -> list[Entity]: + """Keep one entity per physical `schema.table` relation. + + Later entities win, matching the renderer's previous effective behavior: + dbt entities are sorted model-first/source-last, and both wrote the same + file, so a source already replaced a colliding model's markdown. Doing the + selection up front also prevents their cache fingerprints from fighting. + """ + by_slug: dict[str, Entity] = {} + for entity in entities: + previous = by_slug.get(entity.slug) + if previous is not None: + logger.warning( + f" duplicate relation {entity.slug} appears as " + f"{previous.kind} and {entity.kind}; using {entity.kind}" + ) + by_slug[entity.slug] = entity + return list(by_slug.values()) def _inline_map(custom_entities: list["TableCatalog"]) -> dict[str, "TableCatalog"]: diff --git a/src/wlens/render/sample_cache.py b/src/wlens/render/sample_cache.py index 3e1d604..a57bac3 100644 --- a/src/wlens/render/sample_cache.py +++ b/src/wlens/render/sample_cache.py @@ -13,10 +13,15 @@ Cache invalidates when any of these change for an entity: - Set of column names -- Any column's data_type - Any column's PII flag (added/removed `pii: true` in the dbt schema) - The effective obfuscation ruleset (defaults + `output.obfuscate` in wlens.yml) - The cache format `version` + +A column's data type is stored for diagnostics, but deliberately does not +invalidate sample values. dbt/warehouse type metadata can change spelling or +specificity between compilations, and the rendered cell strings do not depend +on it. Treating that metadata as part of the cache key caused needless sample +replacement and noisy diffs. """ from __future__ import annotations @@ -25,6 +30,7 @@ import json import logging import time +from collections.abc import Iterable from pathlib import Path from typing import TYPE_CHECKING @@ -44,11 +50,11 @@ ) -def cache_path(entity: "Entity", cfg: "Config") -> Path: +def cache_path(entity: Entity, cfg: Config) -> Path: return cfg.cache_dir / f"{entity.slug}.json" -def obfuscation_hash(cfg: "Config") -> str: +def obfuscation_hash(cfg: Config) -> str: """sha256 over the effective scrubber config (defaults + user extras).""" rules = compile_rules(cfg.output.obfuscate) payload = [(r.name, r.pattern.pattern, r.replacement) for r in rules] @@ -57,7 +63,7 @@ def obfuscation_hash(cfg: "Config") -> str: ).hexdigest() -def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None: +def load(entity: Entity, cfg: Config, obf_hash: str) -> list[dict] | None: """Return cached rendered rows if the cache is valid; otherwise None.""" path = cache_path(entity, cfg) if not path.exists(): @@ -71,8 +77,20 @@ def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None: return None if data.get("obfuscation_hash") != obf_hash: return None - if _columns_from_cache(data.get("columns") or []) != _columns_view(entity): + + cached_columns = data.get("columns") or [] + cached_names_hash = data.get("column_names_hash") + # Older v1 cache files predate the explicit hash. Derive it from their + # column list so upgrading wlens does not itself churn every sample. + if cached_names_hash is None: + cached_names_hash = _column_names_hash_from_cache(cached_columns) + if cached_names_hash != column_names_hash(entity): + return None + # PII status remains a separate safety check: newly-sensitive values must + # be redacted even though the relation's column names did not change. + if _pii_from_cache(cached_columns) != _pii_view(entity): return None + rows = data.get("rows") if not isinstance(rows, list): return None @@ -80,9 +98,9 @@ def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None: def save( - entity: "Entity", + entity: Entity, rendered_rows: list[dict], - cfg: "Config", + cfg: Config, obf_hash: str, ) -> None: """Write rendered (post-obfuscation) rows to the cache.""" @@ -94,6 +112,7 @@ def save( "schema": entity.schema_name, "table": entity.table_name, "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "column_names_hash": column_names_hash(entity), "obfuscation_hash": obf_hash, "columns": _columns_for_write(entity), "rows": rendered_rows, @@ -101,7 +120,7 @@ def save( path.write_text(json.dumps(payload, indent=2) + "\n") -def _columns_for_write(entity: "Entity") -> list[dict]: +def _columns_for_write(entity: Entity) -> list[dict]: pii = pii_column_names(entity.columns) return [ {"name": c.name, "data_type": c.data_type, "pii": c.name in pii} @@ -109,17 +128,30 @@ def _columns_for_write(entity: "Entity") -> list[dict]: ] -def _columns_view(entity: "Entity") -> dict: +def column_names_hash(entity: Entity) -> str: + """Stable fingerprint of the column-name set (column order is irrelevant).""" + return _hash_column_names(c.name for c in entity.columns.values()) + + +def _column_names_hash_from_cache(cached: list[dict]) -> str: + return _hash_column_names( + c["name"] for c in cached if isinstance(c, dict) and "name" in c + ) + + +def _hash_column_names(names: Iterable[str]) -> str: + payload = sorted(set(names)) + return "sha256:" + hashlib.sha256(json.dumps(payload).encode()).hexdigest() + + +def _pii_view(entity: Entity) -> dict[str, bool]: pii = pii_column_names(entity.columns) - return { - c.name: {"data_type": c.data_type, "pii": c.name in pii} - for c in entity.columns.values() - } + return {c.name: c.name in pii for c in entity.columns.values()} -def _columns_from_cache(cached: list[dict]) -> dict: +def _pii_from_cache(cached: list[dict]) -> dict[str, bool]: return { - c["name"]: {"data_type": c.get("data_type"), "pii": bool(c.get("pii", False))} + c["name"]: bool(c.get("pii", False)) for c in cached if isinstance(c, dict) and "name" in c } diff --git a/tests/test_render_markdown.py b/tests/test_render_markdown.py index aa35978..9b60faf 100644 --- a/tests/test_render_markdown.py +++ b/tests/test_render_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import json import textwrap from pathlib import Path +from wlens.adapters.base import Column, Entity from wlens.adapters.dbt import DbtAdapter from wlens.config import load_config from wlens.entities.loader import load_entities @@ -361,6 +363,41 @@ def _widget_only(entities): return [e for e in entities if e.slug == "prod.dim_widget"] +def test_duplicate_model_and_source_share_one_cache_entry(tmp_path, dbt_project): + """A model/source pair for one physical relation must not overwrite the + same cache twice on every generate run.""" + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + model = _widget_only(DbtAdapter(cfg).list_entities())[0] + source = Entity( + kind="source", + schema_name=model.schema_name, + table_name=model.table_name, + description="Source definition wins, matching the old final output.", + columns={ + "widget_id": Column(name="widget_id"), + # Deliberately differs from the model fingerprint. Before + # deduplication, these two entities invalidated each other. + "source_color": Column(name="source_color"), + }, + ) + executor = _StubExecutor( + headers=["widget_id", "source_color"], + rows=[(1, "red")], + ) + + count = render_and_write_all([model, source], [], executor, cfg) + assert count == 1 + assert executor.calls == 1 + first_cache = (cfg.cache_dir / "prod.dim_widget.json").read_text() + + count = render_and_write_all([model, source], [], executor, cfg) + assert count == 1 + assert executor.calls == 1 + assert (cfg.cache_dir / "prod.dim_widget.json").read_text() == first_cache + assert "(source)" in (cfg.output_dir / "prod.dim_widget.md").read_text() + + def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project): cfg = load_config(_setup_project(tmp_path, dbt_project)) cfg.output.include_sample_rows = True @@ -380,7 +417,7 @@ def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project): assert first_md == second_md -def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project): +def test_sample_cache_hash_ignores_type_and_column_order_drift(tmp_path, dbt_project): cfg = load_config(_setup_project(tmp_path, dbt_project)) cfg.output.include_sample_rows = True entities = _widget_only(DbtAdapter(cfg).list_entities()) @@ -389,12 +426,47 @@ def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project): render_and_write_all(entities, [], executor, cfg) assert executor.calls == 1 - # Mutate a column's data_type — cache should invalidate just for this entity. + cache_file = cfg.cache_dir / "prod.dim_widget.json" + assert json.loads(cache_file.read_text())["column_names_hash"].startswith("sha256:") + + # Neither data types nor manifest column order affect rendered sample cells. + # Keep the cache pinned when only that metadata drifts between dbt compiles. next(iter(entities[0].columns.values())).data_type = "TOTALLY_NEW_TYPE" + entities[0].columns = dict(reversed(entities[0].columns.items())) + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + +def test_sample_cache_refetches_when_column_names_change(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + entities[0].columns["new_column"] = Column(name="new_column", data_type="varchar") render_and_write_all(entities, [], executor, cfg) assert executor.calls == 2 +def test_sample_cache_refetches_when_pii_status_changes(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + entities[0].columns["color"].meta["pii"] = True + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 2 + md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "- `color`: | " in md + + def test_sample_cache_refetches_when_obfuscation_changes(tmp_path, dbt_project): cfg = load_config(_setup_project(tmp_path, dbt_project)) cfg.output.include_sample_rows = True From 3ea5ec0d6a062040f08acff0f4f230dac5ee9172 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Wed, 2 Sep 2026 21:32:49 +0200 Subject: [PATCH 2/2] chore: release v0.4.1 --- pyproject.toml | 2 +- src/wlens/__init__.py | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b178037..f2afb0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "wlens" -version = "0.4.0" +version = "0.4.1" description = "Warehouse lens for AI agents — per-table markdown docs + read-only SQL adaptor, zero-runtime schema exploration for Claude Code, Cursor, Continue." readme = "README.md" license = { file = "LICENSE" } diff --git a/src/wlens/__init__.py b/src/wlens/__init__.py index c766c27..08a7f1d 100644 --- a/src/wlens/__init__.py +++ b/src/wlens/__init__.py @@ -1,3 +1,3 @@ """wlens — warehouse lens for AI agents.""" -__version__ = "0.1.0" +__version__ = "0.4.1" diff --git a/uv.lock b/uv.lock index 12392a3..a5ab60f 100644 --- a/uv.lock +++ b/uv.lock @@ -1159,7 +1159,7 @@ wheels = [ [[package]] name = "wlens" -version = "0.4.0" +version = "0.4.1" source = { editable = "." } dependencies = [ { name = "anyio" },