From 15a993f4fed75890226a54cc4378de65fe575486 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Sat, 23 May 2026 18:48:45 +0200 Subject: [PATCH 1/4] feat(generate): cache sample rows on disk `wlens generate` used to re-query every table for sample rows on every run. The rows came back in different order each time (no ORDER BY), producing hundreds of lines of diff churn, and on big tables the re-query was just slow. Now sample rows are written to wlens/.cache/samples/..json on the first run and reused on every subsequent run. The cache is committed, so teammates and CI share the same pinned rows. We re-fetch when the table's columns change, when its PII tags change, or when the obfuscation rules change. `--refresh-samples` forces a re-fetch (pass no value for all tables, or `=schema.table[,...]` for specific ones). Existing SQL query cache also moves from wlens/cache/sql/ to wlens/.cache/sql/ so both caches live under one wlens-internal root. `wlens init` writes a wlens/.gitignore that keeps the SQL cache out but tracks the sample cache. `wlens generate` migrates an older wlens/.gitignore in place. --- .gitignore | 1 + src/wlens/cli.py | 66 ++++++++++++++-- src/wlens/config.py | 7 ++ src/wlens/executor/base.py | 4 +- src/wlens/render/markdown.py | 121 ++++++++++++++++++++++++------ src/wlens/render/sample_cache.py | 125 +++++++++++++++++++++++++++++++ tests/test_cli.py | 54 ++++++++++++- tests/test_render_markdown.py | 117 +++++++++++++++++++++++++++++ 8 files changed, 461 insertions(+), 34 deletions(-) create mode 100644 src/wlens/render/sample_cache.py diff --git a/.gitignore b/.gitignore index a718ce8..47e9216 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ htmlcov/ .vscode/ .wlens-cache/ +wlens/.cache/sql/ diff --git a/src/wlens/cli.py b/src/wlens/cli.py index a27b12f..3ea72c4 100644 --- a/src/wlens/cli.py +++ b/src/wlens/cli.py @@ -24,6 +24,16 @@ logger = logging.getLogger(__name__) +WLENS_GITIGNORE_BODY = ( + "# wlens-managed — do not edit.\n" + ".cache/*\n" + "!.cache/samples/\n" + "share/\n" +) +# Predecessor body (pre-`.cache/` rename). `wlens generate` rewrites the file +# in-place when it sees this exact content so existing installs migrate. +WLENS_GITIGNORE_LEGACY = "# wlens-managed — do not edit.\ncache/\nshare/\n" + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( @@ -39,6 +49,17 @@ def main(argv: list[str] | None = None) -> int: p_gen.add_argument("--config", help=f"Path to {DEFAULT_CONFIG_FILENAME} (default: walk up from cwd).") p_gen.add_argument("--output-dir", help="Override output directory from config.") p_gen.add_argument("--skip-samples", action="store_true", help="Don't fetch sample rows from the warehouse.") + p_gen.add_argument( + "--refresh-samples", + nargs="?", + const="*", + default=None, + metavar="SLUG[,SLUG…]", + help=( + "Force-refresh sample rows. Pass with no value to refresh every table, " + "or with a comma-separated list of `schema.table` slugs to refresh just those." + ), + ) p_query = sub.add_parser("query", help="Execute a read-only SQL query.") p_query.add_argument("sql", nargs="?", help="SQL to execute (or piped via stdin).") @@ -124,6 +145,7 @@ def main(argv: list[str] | None = None) -> int: config_path=Path(args.config) if args.config else None, output_override=args.output_dir, skip_samples=args.skip_samples, + refresh_samples=args.refresh_samples, ) if args.command == "query": return _cmd_query( @@ -285,12 +307,13 @@ def _cmd_init(*, force: bool) -> int: dest.write_text(content) logger.info(f"wrote {dest.relative_to(cwd)}") - # Seed `wlens/.gitignore` so cache + share stay out of the consumer's repo. - # The generated schema markdown IS meant to be committed — not ignored. + # Seed `wlens/.gitignore`. Schema markdown + sample cache are committed + # (the latter pins sample rows for stable diffs); SQL query cache and + # share/ stay out of the consumer's repo. gitignore = cwd / "wlens" / ".gitignore" if not gitignore.exists() or force: gitignore.parent.mkdir(parents=True, exist_ok=True) - gitignore.write_text("# wlens-managed — do not edit.\ncache/\nshare/\n") + gitignore.write_text(WLENS_GITIGNORE_BODY) logger.info(f"wrote {gitignore.relative_to(cwd)}") logger.info("") @@ -390,7 +413,28 @@ def _set_yaml_value(yaml_content: str, key: str, value: str) -> str: return "".join(out_lines) -def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip_samples: bool) -> int: +def _migrate_wlens_gitignore(repo_root: Path) -> None: + """Rewrite a pre-`.cache/` `wlens/.gitignore` to the new pattern in place. + + One-shot migration helper. Safe to delete (along with `WLENS_GITIGNORE_LEGACY` + and the two test cases in tests/test_cli.py) once existing installs have all + run `wlens generate` at least once after the rename — target 0.5+. + """ + path = repo_root / "wlens" / ".gitignore" + if not path.exists(): + return + if path.read_text() == WLENS_GITIGNORE_LEGACY: + path.write_text(WLENS_GITIGNORE_BODY) + logger.info(f"updated {path.relative_to(repo_root)} for new cache layout") + + +def _cmd_generate( + *, + config_path: Path | None, + output_override: str | None, + skip_samples: bool, + refresh_samples: str | None, +) -> int: from .adapters.dbt import DbtAdapter from .entities.loader import load_entities from .executor import build_executor @@ -400,6 +444,8 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip if output_override: config.output.dir = output_override + _migrate_wlens_gitignore(config.repo_root) + if config.adapter.kind != "dbt": raise NotImplementedError( f"Adapter kind {config.adapter.kind!r} not yet implemented. " @@ -419,7 +465,15 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip logger.warning(f"could not build executor ({e!r}) — continuing without sample rows.") executor = None - count = render_and_write_all(entities, custom_entities, executor, config) + refresh_arg: bool | set[str] = False + if refresh_samples == "*": + refresh_arg = True + elif refresh_samples: + refresh_arg = {s.strip() for s in refresh_samples.split(",") if s.strip()} + + count = render_and_write_all( + entities, custom_entities, executor, config, refresh_samples=refresh_arg + ) logger.info(f"wrote {count} entity files to {config.output.dir}") if executor is not None: executor.close() @@ -504,7 +558,7 @@ def _cmd_clean( ] # Back-compat: previous versions scattered files at the repo root. Clean # them up if they're still there. - for legacy in (".wlens-cache", "wlens-share", ".claude/schema"): + for legacy in (".wlens-cache", "wlens/cache", "wlens-share", ".claude/schema"): legacy_path = repo_root / legacy if legacy_path.exists(): targets.append(legacy_path) diff --git a/src/wlens/config.py b/src/wlens/config.py index 4ab1b70..0033971 100644 --- a/src/wlens/config.py +++ b/src/wlens/config.py @@ -47,6 +47,7 @@ DEFAULT_CONFIG_FILENAME = "wlens.yml" DEFAULT_OUTPUT_DIR = "wlens/schema" +DEFAULT_CACHE_DIR = "wlens/.cache/samples" DEFAULT_SAMPLE_SIZE = 5 ENV_VAR_PATTERN = re.compile(r"\$\{([A-Z_][A-Z0-9_]*)\}") @@ -75,6 +76,7 @@ class ExecutorConfig: @dataclass class OutputConfig: dir: str = DEFAULT_OUTPUT_DIR + cache_dir: str = DEFAULT_CACHE_DIR include_sample_rows: bool = True sample_size: int = DEFAULT_SAMPLE_SIZE # Extra value-level obfuscation rules appended to the built-in defaults @@ -103,6 +105,10 @@ class Config: def output_dir(self) -> Path: return (self.repo_root / self.output.dir).resolve() + @property + def cache_dir(self) -> Path: + return (self.repo_root / self.output.cache_dir).resolve() + def find_config(start: Path | None = None) -> Path: """Walk up from `start` looking for a wlens.yml.""" @@ -160,6 +166,7 @@ def _build_config(raw: dict, repo_root: Path) -> Config: ) output = OutputConfig( dir=output_raw.get("dir", DEFAULT_OUTPUT_DIR), + cache_dir=output_raw.get("cache_dir", DEFAULT_CACHE_DIR), include_sample_rows=bool(output_raw.get("include_sample_rows", True)), sample_size=int(output_raw.get("sample_size", DEFAULT_SAMPLE_SIZE)), obfuscate=list(output_raw.get("obfuscate") or []), diff --git a/src/wlens/executor/base.py b/src/wlens/executor/base.py index 48b6134..8702d89 100644 --- a/src/wlens/executor/base.py +++ b/src/wlens/executor/base.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -CACHE_DIRNAME = "wlens/cache" +CACHE_DIRNAME = "wlens/.cache" CACHE_TTL_SECONDS = 24 * 60 * 60 # 24 hours CACHE_SUBDIR = "sql" @@ -110,7 +110,7 @@ def run( ) -> tuple[list[str], list[tuple], bool]: """Execute a guarded query, return (headers, rows, cache_hit). - Caches to `.wlens-cache/sql/` with a daily TTL. + Caches to `wlens/.cache/sql/` with a daily TTL. """ assert_read_only(sql) if use_cache: diff --git a/src/wlens/render/markdown.py b/src/wlens/render/markdown.py index a726282..d86d3e8 100644 --- a/src/wlens/render/markdown.py +++ b/src/wlens/render/markdown.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Callable from ..adapters.base import Column, Entity, Parent +from . import sample_cache from .obfuscate import compile_rules, obfuscate from .pii import PII_REDACTED, pii_column_names from .preserve import read_manual_block @@ -43,8 +44,16 @@ def render_and_write_all( custom_entities: list["TableCatalog"], executor: "Executor | None", config: "Config", + *, + refresh_samples: bool | set[str] = False, ) -> int: - """Render every entity + the index into `config.output_dir`.""" + """Render every entity + the index into `config.output_dir`. + + `refresh_samples` controls the sample-row cache: + - `False` (default): reuse cache when valid, refetch otherwise. + - `True`: ignore cache for every entity, refetch all. + - `set[str]` of entity slugs: refetch just those, reuse cache for the rest. + """ output_dir = config.output_dir output_dir.mkdir(parents=True, exist_ok=True) @@ -53,12 +62,18 @@ def render_and_write_all( obfuscate_value = functools.partial( obfuscate, rules=compile_rules(config.output.obfuscate) ) + obf_hash = sample_cache.obfuscation_hash(config) + refresh_all = refresh_samples is True + refresh_set: set[str] = ( + set(refresh_samples) if isinstance(refresh_samples, (set, list, tuple)) else set() + ) for entity in entities: - head_rows = _fetch_head(executor, entity, sample_size) if executor else [] - body = _render_entity( - entity, head_rows, inline_map.get(entity.slug), obfuscate_value + rendered_rows = _resolve_sample_rows( + entity, executor, sample_size, config, obfuscate_value, obf_hash, + refresh_all=refresh_all, refresh_set=refresh_set, ) + body = _render_entity(entity, rendered_rows, inline_map.get(entity.slug)) _write_file(output_dir / entity.filename, body) index_path = output_dir / "_index.md" @@ -72,9 +87,8 @@ def render_and_write_all( def _render_entity( entity: Entity, - head_rows: list[dict], + rendered_rows: list[dict], custom_entity: "TableCatalog | None", - obfuscate_value: Callable[[str], str], ) -> str: kind_label = "model" if entity.kind == "model" else "source" if custom_entity is not None: @@ -88,9 +102,7 @@ def _render_entity( if custom_entity is not None: lines.extend(custom_entity.render()) lines.extend(_render_parents(entity.parents)) - lines.extend( - _render_sample_rows(head_rows, pii_column_names(entity.columns), obfuscate_value) - ) + lines.extend(_render_sample_rows(rendered_rows)) lines.extend(_render_compiled_sql(entity.compiled_sql)) return "\n".join(lines).rstrip() + "\n" @@ -125,22 +137,19 @@ def _render_parents(parents: list[Parent]) -> list[str]: return lines -def _render_sample_rows( - head_rows: list[dict], - pii_columns: set[str], - obfuscate_value: Callable[[str], str], -) -> list[str]: - """Column-first (transposed) rendering: one bullet per column, pipe-separated values.""" - if not head_rows: +def _render_sample_rows(rendered_rows: list[dict]) -> list[str]: + """Column-first (transposed) rendering: one bullet per column, pipe-separated values. + + Cells are expected to be already obfuscated + truncated (see + `_eagerly_render_rows`). + """ + if not rendered_rows: return [] - cols = list(head_rows[0].keys()) - n_rows = len(head_rows) + cols = list(rendered_rows[0].keys()) + n_rows = len(rendered_rows) lines = [f"## Sample rows ({n_rows} rows, one line per column)", ""] for c in cols: - if c in pii_columns: - values = " | ".join([PII_REDACTED] * n_rows) - else: - values = " | ".join(_cell(row.get(c), obfuscate_value) for row in head_rows) + values = " | ".join(row.get(c, "") for row in rendered_rows) lines.append(f"- `{c}`: {values}") lines.append("") return lines @@ -175,7 +184,52 @@ def _render_index(entities: list[Entity]) -> str: return "\n".join(lines).rstrip() + "\n" -# ─── Sample-row fetching ──────────────────────────────────────────────────── +# ─── Sample-row fetching + caching ────────────────────────────────────────── + + +def _resolve_sample_rows( + entity: Entity, + executor: "Executor | None", + sample_size: int, + cfg: "Config", + obfuscate_value: Callable[[str], str], + obf_hash: str, + *, + refresh_all: bool, + refresh_set: set[str], +) -> list[dict]: + """Return rendered (post-obfuscation) rows for an entity, using cache when possible.""" + if sample_size <= 0: + return [] + + force_refresh = refresh_all or entity.slug in refresh_set + if not force_refresh: + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + return cached + + if executor is None: + # No way to fetch fresh. Fall back to whatever cache exists, even if + # the user asked for refresh — they can't have it without a DB. + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + if force_refresh: + logger.warning( + f" refresh requested for {entity.slug} but no executor configured; " + "reusing cached samples" + ) + return cached + return [] + + raw = _fetch_head(executor, entity, sample_size) + if not raw: + return [] + rendered = _eagerly_render_rows(raw, entity, obfuscate_value) + try: + sample_cache.save(entity, rendered, cfg, obf_hash) + except OSError as e: + logger.warning(f" could not write sample cache for {entity.slug}: {e!r}") + return rendered def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[dict]: @@ -183,7 +237,7 @@ def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[ return [] try: headers, rows, _ = executor.run_direct( - f'select * from {entity.schema_name}.{entity.table_name} limit {sample_size}' + f"select * from {entity.schema_name}.{entity.table_name} limit {sample_size}" ) except Exception as e: logger.warning(f" could not fetch samples for {entity.slug}: {e!r}") @@ -194,6 +248,25 @@ def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[ ] +def _eagerly_render_rows( + head_rows: list[dict], + entity: Entity, + obfuscate_value: Callable[[str], str], +) -> list[dict]: + """Convert raw fetched rows into pre-obfuscated, cell-truncated dicts.""" + pii_cols = pii_column_names(entity.columns) + rendered: list[dict] = [] + for row in head_rows: + rendered_row: dict = {} + for col in row.keys(): + if col in pii_cols: + rendered_row[col] = PII_REDACTED + else: + rendered_row[col] = _cell(row.get(col), obfuscate_value) + rendered.append(rendered_row) + return rendered + + # ─── File writing + manual-notes preservation ─────────────────────────────── diff --git a/src/wlens/render/sample_cache.py b/src/wlens/render/sample_cache.py new file mode 100644 index 0000000..3e1d604 --- /dev/null +++ b/src/wlens/render/sample_cache.py @@ -0,0 +1,125 @@ +"""On-disk cache for sample rows rendered into per-table markdown. + +Goal: skip the warehouse roundtrip on subsequent `wlens generate` runs when +nothing schema-relevant has changed, and pin the rendered rows so they don't +churn the git diff between runs. + +Cache files live under `/.
.json` (default +`wlens/.cache/samples/`). Each file stores the **post-obfuscation, +post-truncation** cell strings — exactly what gets rendered into the .md — +so the cache itself adds zero new PII surface vs. the markdown that's +already committed. + +Cache invalidates when any of these change for an entity: + +- Set of column names +- Any column's data_type +- Any column's PII flag (added/removed `pii: true` in the dbt schema) +- The effective obfuscation ruleset (defaults + `output.obfuscate` in wlens.yml) +- The cache format `version` +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +from pathlib import Path +from typing import TYPE_CHECKING + +from .obfuscate import compile_rules +from .pii import pii_column_names + +if TYPE_CHECKING: + from ..adapters.base import Entity + from ..config import Config + +logger = logging.getLogger(__name__) + +CACHE_VERSION = 1 +_DOC = ( + "wlens-managed sample cache. Schema docs live in wlens/schema/. " + "Do not read this file — read the .md instead." +) + + +def cache_path(entity: "Entity", cfg: "Config") -> Path: + return cfg.cache_dir / f"{entity.slug}.json" + + +def obfuscation_hash(cfg: "Config") -> str: + """sha256 over the effective scrubber config (defaults + user extras).""" + rules = compile_rules(cfg.output.obfuscate) + payload = [(r.name, r.pattern.pattern, r.replacement) for r in rules] + return "sha256:" + hashlib.sha256( + json.dumps(payload, sort_keys=True).encode() + ).hexdigest() + + +def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None: + """Return cached rendered rows if the cache is valid; otherwise None.""" + path = cache_path(entity, cfg) + if not path.exists(): + return None + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError) as e: + logger.warning(f" sample cache for {entity.slug} unreadable ({e}); refetching") + return None + if data.get("version") != CACHE_VERSION: + return None + if data.get("obfuscation_hash") != obf_hash: + return None + if _columns_from_cache(data.get("columns") or []) != _columns_view(entity): + return None + rows = data.get("rows") + if not isinstance(rows, list): + return None + return rows + + +def save( + entity: "Entity", + rendered_rows: list[dict], + cfg: "Config", + obf_hash: str, +) -> None: + """Write rendered (post-obfuscation) rows to the cache.""" + path = cache_path(entity, cfg) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "_doc": _DOC, + "version": CACHE_VERSION, + "schema": entity.schema_name, + "table": entity.table_name, + "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "obfuscation_hash": obf_hash, + "columns": _columns_for_write(entity), + "rows": rendered_rows, + } + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _columns_for_write(entity: "Entity") -> list[dict]: + pii = pii_column_names(entity.columns) + return [ + {"name": c.name, "data_type": c.data_type, "pii": c.name in pii} + for c in entity.columns.values() + ] + + +def _columns_view(entity: "Entity") -> dict: + pii = pii_column_names(entity.columns) + return { + c.name: {"data_type": c.data_type, "pii": c.name in pii} + for c in entity.columns.values() + } + + +def _columns_from_cache(cached: list[dict]) -> dict: + return { + c["name"]: {"data_type": c.get("data_type"), "pii": bool(c.get("pii", False))} + for c in cached + if isinstance(c, dict) and "name" in c + } diff --git a/tests/test_cli.py b/tests/test_cli.py index c6613b6..c3d7039 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -60,12 +60,62 @@ def test_init_writes_wlens_gitignore(tmp_path: Path, monkeypatch): gi = tmp_path / "wlens" / ".gitignore" assert gi.exists() body = gi.read_text() - # cache/ and share/ must be ignored — schema/ must NOT be (it's the point). - assert "cache/" in body + # SQL query cache + share/ ignored; sample-row cache explicitly tracked; + # generated schema markdown stays out of the ignore list (it's the point). + assert ".cache/*" in body + assert "!.cache/samples/" in body assert "share/" in body assert "schema/" not in body +def test_generate_migrates_legacy_wlens_gitignore(tmp_path: Path, monkeypatch, dbt_project): + """An older `wlens/.gitignore` (pre-`.cache/` rename) is rewritten in place.""" + import textwrap + from wlens.cli import WLENS_GITIGNORE_BODY, WLENS_GITIGNORE_LEGACY + + monkeypatch.chdir(tmp_path) + (tmp_path / "wlens.yml").write_text( + textwrap.dedent(f""" + adapter: + kind: dbt + project_dir: {dbt_project.relative_to(tmp_path)} + output: + dir: wlens/schema + include_sample_rows: false + """).lstrip() + ) + gi = tmp_path / "wlens" / ".gitignore" + gi.parent.mkdir(parents=True, exist_ok=True) + gi.write_text(WLENS_GITIGNORE_LEGACY) + + assert main(["generate"]) == 0 + assert gi.read_text() == WLENS_GITIGNORE_BODY + + +def test_generate_leaves_custom_wlens_gitignore_alone(tmp_path: Path, monkeypatch, dbt_project): + """If the user has hand-edited `wlens/.gitignore`, don't touch it.""" + import textwrap + + monkeypatch.chdir(tmp_path) + (tmp_path / "wlens.yml").write_text( + textwrap.dedent(f""" + adapter: + kind: dbt + project_dir: {dbt_project.relative_to(tmp_path)} + output: + dir: wlens/schema + include_sample_rows: false + """).lstrip() + ) + gi = tmp_path / "wlens" / ".gitignore" + gi.parent.mkdir(parents=True, exist_ok=True) + custom = "# my own rules\nfoo/\nbar/\n" + gi.write_text(custom) + + assert main(["generate"]) == 0 + assert gi.read_text() == custom + + def test_init_defaults_project_dir_to_dot_when_no_dbt_project(tmp_path: Path, monkeypatch): monkeypatch.chdir(tmp_path) main(["init"]) diff --git a/tests/test_render_markdown.py b/tests/test_render_markdown.py index b7ed2e2..31b2b91 100644 --- a/tests/test_render_markdown.py +++ b/tests/test_render_markdown.py @@ -35,8 +35,10 @@ class _StubExecutor: def __init__(self, headers: list[str], rows: list[tuple]): self._headers = headers self._rows = rows + self.calls = 0 def run_direct(self, sql: str): # noqa: ARG002 — sql ignored, fixed payload + self.calls += 1 return self._headers, self._rows, False @@ -343,3 +345,118 @@ def test_plans_example_renders(): # auto-render still handles plain scalars. assert "**Price:** $0" in md assert "**Seats included:** 5" in md + + +# ─── Sample-row cache ─────────────────────────────────────────────────────── + + +def _widget_executor() -> _StubExecutor: + return _StubExecutor( + headers=["widget_id", "color"], + rows=[(1, "red"), (2, "blue")], + ) + + +def _widget_only(entities): + return [e for e in entities if e.slug == "prod.dim_widget"] + + +def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + first_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert executor.calls == 1 + cache_file = cfg.cache_dir / "prod.dim_widget.json" + assert cache_file.exists() + + # Second run: same executor, same cache → zero new queries, identical output. + render_and_write_all(entities, [], executor, cfg) + second_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert executor.calls == 1, "cache hit must not re-query" + assert first_md == second_md + + +def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + # Mutate a column's data_type — cache should invalidate just for this entity. + next(iter(entities[0].columns.values())).data_type = "TOTALLY_NEW_TYPE" + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 2 + + +def test_sample_cache_refetches_when_obfuscation_changes(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + cfg.output.obfuscate = [{"pattern": "red", "replacement": ""}] + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 2 + md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "" in md + + +def test_refresh_samples_all_forces_requery(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + render_and_write_all(entities, [], executor, cfg, refresh_samples=True) + assert executor.calls == 2 + + +def test_refresh_samples_slug_targets_one_entity(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = [ + e for e in DbtAdapter(cfg).list_entities() + if e.slug in {"prod.dim_widget", "prod.fct_sale"} + ] + executor = _StubExecutor(headers=["a"], rows=[(1,)]) + + render_and_write_all(entities, [], executor, cfg) + base_calls = executor.calls + assert base_calls >= 2 # one per entity on cold cache + + render_and_write_all( + entities, [], executor, cfg, refresh_samples={"prod.dim_widget"} + ) + # Only one entity refetched; the other should hit the cache. + assert executor.calls == base_calls + 1 + + +def test_sample_cache_reused_when_no_executor(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + md_with_executor = (cfg.output_dir / "prod.dim_widget.md").read_text() + + # Wipe the markdown so we know the second render had to build it from scratch. + (cfg.output_dir / "prod.dim_widget.md").unlink() + + render_and_write_all(entities, [], None, cfg) + md_offline = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "## Sample rows" in md_offline + assert md_offline == md_with_executor From 79102b89c2ec581ac8d81fe36107a0ce8c597e37 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Sat, 23 May 2026 18:51:42 +0200 Subject: [PATCH 2/4] chore: release v0.3.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 705d651..467f868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "wlens" -version = "0.2.0" +version = "0.3.0" description = "Warehouse lens for AI agents — per-table markdown docs + read-only SQL adaptor, zero-runtime schema exploration for Claude Code, Cursor, Continue." readme = "README.md" license = { file = "LICENSE" } From 87ca76b715adda6aef509acb96869638b7e82c7d Mon Sep 17 00:00:00 2001 From: David Baussart Date: Sat, 23 May 2026 19:09:36 +0200 Subject: [PATCH 3/4] fix(generate): keep cache when forced refresh comes back empty `--refresh-samples` followed by an empty fetch (transient query error or genuinely-empty table) used to drop the sample-rows section to [] without touching the cache file. Next run would silently put the old rows back, so the .md flipped between "with rows" and "without rows" across runs. Now an empty fetch falls back to the existing cache and logs a warning. Disk state and the rendered markdown stay in sync. --- src/wlens/render/markdown.py | 10 ++++++++++ tests/test_render_markdown.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/wlens/render/markdown.py b/src/wlens/render/markdown.py index d86d3e8..27666d3 100644 --- a/src/wlens/render/markdown.py +++ b/src/wlens/render/markdown.py @@ -223,6 +223,16 @@ def _resolve_sample_rows( raw = _fetch_head(executor, entity, sample_size) if not raw: + # Fetch failed or table is empty. Don't drop the rendered sample-rows + # section to `[]` if we already have a valid cache — that would flip + # the .md between "with rows" and "without rows" across runs whenever + # a query is transiently flaky. + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + logger.warning( + f" sample fetch for {entity.slug} returned no rows; reusing cached samples" + ) + return cached return [] rendered = _eagerly_render_rows(raw, entity, obfuscate_value) try: diff --git a/tests/test_render_markdown.py b/tests/test_render_markdown.py index 31b2b91..aa35978 100644 --- a/tests/test_render_markdown.py +++ b/tests/test_render_markdown.py @@ -444,6 +444,37 @@ def test_refresh_samples_slug_targets_one_entity(tmp_path, dbt_project): assert executor.calls == base_calls + 1 +class _FlakyExecutor(_StubExecutor): + """Returns headers/rows on the first call, then empty on subsequent calls.""" + + def run_direct(self, sql: str): # noqa: ARG002 + self.calls += 1 + if self.calls == 1: + return self._headers, self._rows, False + return self._headers, [], False + + +def test_refresh_falls_back_to_cache_when_fetch_returns_empty(tmp_path, dbt_project): + """Forced refresh that comes back empty must not drop the cache or the + rendered section — the next run should still produce identical .md.""" + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + + executor = _FlakyExecutor(headers=["widget_id", "color"], rows=[(1, "red"), (2, "blue")]) + + render_and_write_all(entities, [], executor, cfg) + first_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "## Sample rows" in first_md + + # Second run: --refresh-samples forces a fetch, but the executor returns + # nothing. The .md must still contain the previously-cached rows. + render_and_write_all(entities, [], executor, cfg, refresh_samples=True) + second_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert second_md == first_md + assert executor.calls == 2 # the refresh did try to fetch + + def test_sample_cache_reused_when_no_executor(tmp_path, dbt_project): cfg = load_config(_setup_project(tmp_path, dbt_project)) cfg.output.include_sample_rows = True From bfddc74073736c17ae1ff1f030b8f76195405b9c Mon Sep 17 00:00:00 2001 From: David Baussart Date: Sat, 23 May 2026 19:09:36 +0200 Subject: [PATCH 4/4] chore: update uv.lock for v0.3.0 --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index f81a924..b9b1cda 100644 --- a/uv.lock +++ b/uv.lock @@ -1159,7 +1159,7 @@ wheels = [ [[package]] name = "wlens" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "anyio" },