Fix sample cache churn and release v0.4.1

docs/quickstart.md

@@ -139,6 +139,12 @@ wlens/schema/
 If your executor is configured and reachable, each file includes five
 sample rows. Offline / no credentials? Pass `--skip-samples`.
 
+Sample rows are pinned in `wlens/.cache/samples/` to keep generated diffs
+stable. `wlens generate` reuses them while the table's column-name hash is
+unchanged (column order and data-type metadata do not count). PII flags and
+obfuscation rules are checked separately so newly-sensitive values cannot leak.
+Use `--refresh-samples` only when you intentionally want new samples.
+
 ## 6. Ask your AI agent something
 
 Open Claude Code, Cursor, Continue, or Codex and ask a question. The

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "wlens"
-version = "0.4.0"
+version = "0.4.1"
 description = "Warehouse lens for AI agents — per-table markdown docs + read-only SQL adaptor, zero-runtime schema exploration for Claude Code, Cursor, Continue."
 readme = "README.md"
 license = { file = "LICENSE" }

src/wlens/__init__.py

@@ -1,3 +1,3 @@
 """wlens — warehouse lens for AI agents."""
 
-__version__ = "0.1.0"
+__version__ = "0.4.1"

src/wlens/render/markdown.py

@@ -57,6 +57,11 @@ def render_and_write_all(
     output_dir = config.output_dir
     output_dir.mkdir(parents=True, exist_ok=True)
 
+    # A dbt manifest can expose the same physical relation as both a model and
+    # a source. Both entities otherwise share a markdown path and sample-cache
+    # path, causing them to invalidate and overwrite each other on every run.
+    entities = _deduplicate_entities(entities)
+
     inline_map = _inline_map(custom_entities)
     sample_size = config.output.sample_size if config.output.include_sample_rows else 0
     obfuscate_value = functools.partial(
@@ -287,7 +292,27 @@ def _write_file(path: Path, new_body: str) -> None:
     logger.info(f"  wrote {path.name}")
 
 
-# ─── Inlining custom entities ───────────────────────────────────────────────
+# ─── Duplicate relations + inlining custom entities ────────────────────────
+
+
+def _deduplicate_entities(entities: list[Entity]) -> list[Entity]:
+    """Keep one entity per physical `schema.table` relation.
+
+    Later entities win, matching the renderer's previous effective behavior:
+    dbt entities are sorted model-first/source-last, and both wrote the same
+    file, so a source already replaced a colliding model's markdown. Doing the
+    selection up front also prevents their cache fingerprints from fighting.
+    """
+    by_slug: dict[str, Entity] = {}
+    for entity in entities:
+        previous = by_slug.get(entity.slug)
+        if previous is not None:
+            logger.warning(
+                f"  duplicate relation {entity.slug} appears as "
+                f"{previous.kind} and {entity.kind}; using {entity.kind}"
+            )
+        by_slug[entity.slug] = entity
+    return list(by_slug.values())
 
 
 def _inline_map(custom_entities: list["TableCatalog"]) -> dict[str, "TableCatalog"]:

src/wlens/render/sample_cache.py

@@ -13,10 +13,15 @@
 Cache invalidates when any of these change for an entity:
 
 - Set of column names
-- Any column's data_type
 - Any column's PII flag (added/removed `pii: true` in the dbt schema)
 - The effective obfuscation ruleset (defaults + `output.obfuscate` in wlens.yml)
 - The cache format `version`
+
+A column's data type is stored for diagnostics, but deliberately does not
+invalidate sample values. dbt/warehouse type metadata can change spelling or
+specificity between compilations, and the rendered cell strings do not depend
+on it. Treating that metadata as part of the cache key caused needless sample
+replacement and noisy diffs.
 """
 
 from __future__ import annotations
@@ -25,6 +30,7 @@
 import json
 import logging
 import time
+from collections.abc import Iterable
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -44,11 +50,11 @@
 )
 
 
-def cache_path(entity: "Entity", cfg: "Config") -> Path:
+def cache_path(entity: Entity, cfg: Config) -> Path:
     return cfg.cache_dir / f"{entity.slug}.json"
 
 
-def obfuscation_hash(cfg: "Config") -> str:
+def obfuscation_hash(cfg: Config) -> str:
     """sha256 over the effective scrubber config (defaults + user extras)."""
     rules = compile_rules(cfg.output.obfuscate)
     payload = [(r.name, r.pattern.pattern, r.replacement) for r in rules]
@@ -57,7 +63,7 @@ def obfuscation_hash(cfg: "Config") -> str:
     ).hexdigest()
 
 
-def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None:
+def load(entity: Entity, cfg: Config, obf_hash: str) -> list[dict] | None:
     """Return cached rendered rows if the cache is valid; otherwise None."""
     path = cache_path(entity, cfg)
     if not path.exists():
@@ -71,18 +77,30 @@ def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None:
         return None
     if data.get("obfuscation_hash") != obf_hash:
         return None
-    if _columns_from_cache(data.get("columns") or []) != _columns_view(entity):
+
+    cached_columns = data.get("columns") or []
+    cached_names_hash = data.get("column_names_hash")
+    # Older v1 cache files predate the explicit hash. Derive it from their
+    # column list so upgrading wlens does not itself churn every sample.
+    if cached_names_hash is None:
+        cached_names_hash = _column_names_hash_from_cache(cached_columns)
+    if cached_names_hash != column_names_hash(entity):
+        return None
+    # PII status remains a separate safety check: newly-sensitive values must
+    # be redacted even though the relation's column names did not change.
+    if _pii_from_cache(cached_columns) != _pii_view(entity):
         return None
+
     rows = data.get("rows")
     if not isinstance(rows, list):
         return None
     return rows
 
 
 def save(
-    entity: "Entity",
+    entity: Entity,
     rendered_rows: list[dict],
-    cfg: "Config",
+    cfg: Config,
     obf_hash: str,
 ) -> None:
     """Write rendered (post-obfuscation) rows to the cache."""
@@ -94,32 +112,46 @@ def save(
         "schema": entity.schema_name,
         "table": entity.table_name,
         "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "column_names_hash": column_names_hash(entity),
         "obfuscation_hash": obf_hash,
         "columns": _columns_for_write(entity),
         "rows": rendered_rows,
     }
     path.write_text(json.dumps(payload, indent=2) + "\n")
 
 
-def _columns_for_write(entity: "Entity") -> list[dict]:
+def _columns_for_write(entity: Entity) -> list[dict]:
     pii = pii_column_names(entity.columns)
     return [
         {"name": c.name, "data_type": c.data_type, "pii": c.name in pii}
         for c in entity.columns.values()
     ]
 
 
-def _columns_view(entity: "Entity") -> dict:
+def column_names_hash(entity: Entity) -> str:
+    """Stable fingerprint of the column-name set (column order is irrelevant)."""
+    return _hash_column_names(c.name for c in entity.columns.values())
+
+
+def _column_names_hash_from_cache(cached: list[dict]) -> str:
+    return _hash_column_names(
+        c["name"] for c in cached if isinstance(c, dict) and "name" in c
+    )
+
+
+def _hash_column_names(names: Iterable[str]) -> str:
+    payload = sorted(set(names))
+    return "sha256:" + hashlib.sha256(json.dumps(payload).encode()).hexdigest()
+
+
+def _pii_view(entity: Entity) -> dict[str, bool]:
     pii = pii_column_names(entity.columns)
-    return {
-        c.name: {"data_type": c.data_type, "pii": c.name in pii}
-        for c in entity.columns.values()
-    }
+    return {c.name: c.name in pii for c in entity.columns.values()}
 
 
-def _columns_from_cache(cached: list[dict]) -> dict:
+def _pii_from_cache(cached: list[dict]) -> dict[str, bool]:
     return {
-        c["name"]: {"data_type": c.get("data_type"), "pii": bool(c.get("pii", False))}
+        c["name"]: bool(c.get("pii", False))
         for c in cached
         if isinstance(c, dict) and "name" in c
     }

tests/test_render_markdown.py

@@ -2,9 +2,11 @@
 
 from __future__ import annotations
 
+import json
 import textwrap
 from pathlib import Path
 
+from wlens.adapters.base import Column, Entity
 from wlens.adapters.dbt import DbtAdapter
 from wlens.config import load_config
 from wlens.entities.loader import load_entities
@@ -361,6 +363,41 @@ def _widget_only(entities):
     return [e for e in entities if e.slug == "prod.dim_widget"]
 
 
+def test_duplicate_model_and_source_share_one_cache_entry(tmp_path, dbt_project):
+    """A model/source pair for one physical relation must not overwrite the
+    same cache twice on every generate run."""
+    cfg = load_config(_setup_project(tmp_path, dbt_project))
+    cfg.output.include_sample_rows = True
+    model = _widget_only(DbtAdapter(cfg).list_entities())[0]
+    source = Entity(
+        kind="source",
+        schema_name=model.schema_name,
+        table_name=model.table_name,
+        description="Source definition wins, matching the old final output.",
+        columns={
+            "widget_id": Column(name="widget_id"),
+            # Deliberately differs from the model fingerprint. Before
+            # deduplication, these two entities invalidated each other.
+            "source_color": Column(name="source_color"),
+        },
+    )
+    executor = _StubExecutor(
+        headers=["widget_id", "source_color"],
+        rows=[(1, "red")],
+    )
+
+    count = render_and_write_all([model, source], [], executor, cfg)
+    assert count == 1
+    assert executor.calls == 1
+    first_cache = (cfg.cache_dir / "prod.dim_widget.json").read_text()
+
+    count = render_and_write_all([model, source], [], executor, cfg)
+    assert count == 1
+    assert executor.calls == 1
+    assert (cfg.cache_dir / "prod.dim_widget.json").read_text() == first_cache
+    assert "(source)" in (cfg.output_dir / "prod.dim_widget.md").read_text()
+
+
 def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project):
     cfg = load_config(_setup_project(tmp_path, dbt_project))
     cfg.output.include_sample_rows = True
@@ -380,7 +417,7 @@ def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project):
     assert first_md == second_md
 
 
-def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project):
+def test_sample_cache_hash_ignores_type_and_column_order_drift(tmp_path, dbt_project):
     cfg = load_config(_setup_project(tmp_path, dbt_project))
     cfg.output.include_sample_rows = True
     entities = _widget_only(DbtAdapter(cfg).list_entities())
@@ -389,12 +426,47 @@ def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project):
     render_and_write_all(entities, [], executor, cfg)
     assert executor.calls == 1
 
-    # Mutate a column's data_type — cache should invalidate just for this entity.
+    cache_file = cfg.cache_dir / "prod.dim_widget.json"
+    assert json.loads(cache_file.read_text())["column_names_hash"].startswith("sha256:")
+
+    # Neither data types nor manifest column order affect rendered sample cells.
+    # Keep the cache pinned when only that metadata drifts between dbt compiles.
     next(iter(entities[0].columns.values())).data_type = "TOTALLY_NEW_TYPE"
+    entities[0].columns = dict(reversed(entities[0].columns.items()))
+    render_and_write_all(entities, [], executor, cfg)
+    assert executor.calls == 1
+
+
+def test_sample_cache_refetches_when_column_names_change(tmp_path, dbt_project):
+    cfg = load_config(_setup_project(tmp_path, dbt_project))
+    cfg.output.include_sample_rows = True
+    entities = _widget_only(DbtAdapter(cfg).list_entities())
+    executor = _widget_executor()
+
+    render_and_write_all(entities, [], executor, cfg)
+    assert executor.calls == 1
+
+    entities[0].columns["new_column"] = Column(name="new_column", data_type="varchar")
     render_and_write_all(entities, [], executor, cfg)
     assert executor.calls == 2
 
 
+def test_sample_cache_refetches_when_pii_status_changes(tmp_path, dbt_project):
+    cfg = load_config(_setup_project(tmp_path, dbt_project))
+    cfg.output.include_sample_rows = True
+    entities = _widget_only(DbtAdapter(cfg).list_entities())
+    executor = _widget_executor()
+
+    render_and_write_all(entities, [], executor, cfg)
+    assert executor.calls == 1
+
+    entities[0].columns["color"].meta["pii"] = True
+    render_and_write_all(entities, [], executor, cfg)
+    assert executor.calls == 2
+    md = (cfg.output_dir / "prod.dim_widget.md").read_text()
+    assert "- `color`: <pii> | <pii>" in md
+
+
 def test_sample_cache_refetches_when_obfuscation_changes(tmp_path, dbt_project):
     cfg = load_config(_setup_project(tmp_path, dbt_project))
     cfg.output.include_sample_rows = True