diff --git a/.gitignore b/.gitignore index a718ce8..47e9216 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ htmlcov/ .vscode/ .wlens-cache/ +wlens/.cache/sql/ diff --git a/pyproject.toml b/pyproject.toml index 705d651..467f868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "wlens" -version = "0.2.0" +version = "0.3.0" description = "Warehouse lens for AI agents — per-table markdown docs + read-only SQL adaptor, zero-runtime schema exploration for Claude Code, Cursor, Continue." readme = "README.md" license = { file = "LICENSE" } diff --git a/src/wlens/cli.py b/src/wlens/cli.py index a27b12f..3ea72c4 100644 --- a/src/wlens/cli.py +++ b/src/wlens/cli.py @@ -24,6 +24,16 @@ logger = logging.getLogger(__name__) +WLENS_GITIGNORE_BODY = ( + "# wlens-managed — do not edit.\n" + ".cache/*\n" + "!.cache/samples/\n" + "share/\n" +) +# Predecessor body (pre-`.cache/` rename). `wlens generate` rewrites the file +# in-place when it sees this exact content so existing installs migrate. +WLENS_GITIGNORE_LEGACY = "# wlens-managed — do not edit.\ncache/\nshare/\n" + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( @@ -39,6 +49,17 @@ def main(argv: list[str] | None = None) -> int: p_gen.add_argument("--config", help=f"Path to {DEFAULT_CONFIG_FILENAME} (default: walk up from cwd).") p_gen.add_argument("--output-dir", help="Override output directory from config.") p_gen.add_argument("--skip-samples", action="store_true", help="Don't fetch sample rows from the warehouse.") + p_gen.add_argument( + "--refresh-samples", + nargs="?", + const="*", + default=None, + metavar="SLUG[,SLUG…]", + help=( + "Force-refresh sample rows. Pass with no value to refresh every table, " + "or with a comma-separated list of `schema.table` slugs to refresh just those." + ), + ) p_query = sub.add_parser("query", help="Execute a read-only SQL query.") p_query.add_argument("sql", nargs="?", help="SQL to execute (or piped via stdin).") @@ -124,6 +145,7 @@ def main(argv: list[str] | None = None) -> int: config_path=Path(args.config) if args.config else None, output_override=args.output_dir, skip_samples=args.skip_samples, + refresh_samples=args.refresh_samples, ) if args.command == "query": return _cmd_query( @@ -285,12 +307,13 @@ def _cmd_init(*, force: bool) -> int: dest.write_text(content) logger.info(f"wrote {dest.relative_to(cwd)}") - # Seed `wlens/.gitignore` so cache + share stay out of the consumer's repo. - # The generated schema markdown IS meant to be committed — not ignored. + # Seed `wlens/.gitignore`. Schema markdown + sample cache are committed + # (the latter pins sample rows for stable diffs); SQL query cache and + # share/ stay out of the consumer's repo. gitignore = cwd / "wlens" / ".gitignore" if not gitignore.exists() or force: gitignore.parent.mkdir(parents=True, exist_ok=True) - gitignore.write_text("# wlens-managed — do not edit.\ncache/\nshare/\n") + gitignore.write_text(WLENS_GITIGNORE_BODY) logger.info(f"wrote {gitignore.relative_to(cwd)}") logger.info("") @@ -390,7 +413,28 @@ def _set_yaml_value(yaml_content: str, key: str, value: str) -> str: return "".join(out_lines) -def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip_samples: bool) -> int: +def _migrate_wlens_gitignore(repo_root: Path) -> None: + """Rewrite a pre-`.cache/` `wlens/.gitignore` to the new pattern in place. + + One-shot migration helper. Safe to delete (along with `WLENS_GITIGNORE_LEGACY` + and the two test cases in tests/test_cli.py) once existing installs have all + run `wlens generate` at least once after the rename — target 0.5+. + """ + path = repo_root / "wlens" / ".gitignore" + if not path.exists(): + return + if path.read_text() == WLENS_GITIGNORE_LEGACY: + path.write_text(WLENS_GITIGNORE_BODY) + logger.info(f"updated {path.relative_to(repo_root)} for new cache layout") + + +def _cmd_generate( + *, + config_path: Path | None, + output_override: str | None, + skip_samples: bool, + refresh_samples: str | None, +) -> int: from .adapters.dbt import DbtAdapter from .entities.loader import load_entities from .executor import build_executor @@ -400,6 +444,8 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip if output_override: config.output.dir = output_override + _migrate_wlens_gitignore(config.repo_root) + if config.adapter.kind != "dbt": raise NotImplementedError( f"Adapter kind {config.adapter.kind!r} not yet implemented. " @@ -419,7 +465,15 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip logger.warning(f"could not build executor ({e!r}) — continuing without sample rows.") executor = None - count = render_and_write_all(entities, custom_entities, executor, config) + refresh_arg: bool | set[str] = False + if refresh_samples == "*": + refresh_arg = True + elif refresh_samples: + refresh_arg = {s.strip() for s in refresh_samples.split(",") if s.strip()} + + count = render_and_write_all( + entities, custom_entities, executor, config, refresh_samples=refresh_arg + ) logger.info(f"wrote {count} entity files to {config.output.dir}") if executor is not None: executor.close() @@ -504,7 +558,7 @@ def _cmd_clean( ] # Back-compat: previous versions scattered files at the repo root. Clean # them up if they're still there. - for legacy in (".wlens-cache", "wlens-share", ".claude/schema"): + for legacy in (".wlens-cache", "wlens/cache", "wlens-share", ".claude/schema"): legacy_path = repo_root / legacy if legacy_path.exists(): targets.append(legacy_path) diff --git a/src/wlens/config.py b/src/wlens/config.py index 4ab1b70..0033971 100644 --- a/src/wlens/config.py +++ b/src/wlens/config.py @@ -47,6 +47,7 @@ DEFAULT_CONFIG_FILENAME = "wlens.yml" DEFAULT_OUTPUT_DIR = "wlens/schema" +DEFAULT_CACHE_DIR = "wlens/.cache/samples" DEFAULT_SAMPLE_SIZE = 5 ENV_VAR_PATTERN = re.compile(r"\$\{([A-Z_][A-Z0-9_]*)\}") @@ -75,6 +76,7 @@ class ExecutorConfig: @dataclass class OutputConfig: dir: str = DEFAULT_OUTPUT_DIR + cache_dir: str = DEFAULT_CACHE_DIR include_sample_rows: bool = True sample_size: int = DEFAULT_SAMPLE_SIZE # Extra value-level obfuscation rules appended to the built-in defaults @@ -103,6 +105,10 @@ class Config: def output_dir(self) -> Path: return (self.repo_root / self.output.dir).resolve() + @property + def cache_dir(self) -> Path: + return (self.repo_root / self.output.cache_dir).resolve() + def find_config(start: Path | None = None) -> Path: """Walk up from `start` looking for a wlens.yml.""" @@ -160,6 +166,7 @@ def _build_config(raw: dict, repo_root: Path) -> Config: ) output = OutputConfig( dir=output_raw.get("dir", DEFAULT_OUTPUT_DIR), + cache_dir=output_raw.get("cache_dir", DEFAULT_CACHE_DIR), include_sample_rows=bool(output_raw.get("include_sample_rows", True)), sample_size=int(output_raw.get("sample_size", DEFAULT_SAMPLE_SIZE)), obfuscate=list(output_raw.get("obfuscate") or []), diff --git a/src/wlens/executor/base.py b/src/wlens/executor/base.py index 48b6134..8702d89 100644 --- a/src/wlens/executor/base.py +++ b/src/wlens/executor/base.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -CACHE_DIRNAME = "wlens/cache" +CACHE_DIRNAME = "wlens/.cache" CACHE_TTL_SECONDS = 24 * 60 * 60 # 24 hours CACHE_SUBDIR = "sql" @@ -110,7 +110,7 @@ def run( ) -> tuple[list[str], list[tuple], bool]: """Execute a guarded query, return (headers, rows, cache_hit). - Caches to `.wlens-cache/sql/` with a daily TTL. + Caches to `wlens/.cache/sql/` with a daily TTL. """ assert_read_only(sql) if use_cache: diff --git a/src/wlens/render/markdown.py b/src/wlens/render/markdown.py index a726282..27666d3 100644 --- a/src/wlens/render/markdown.py +++ b/src/wlens/render/markdown.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Callable from ..adapters.base import Column, Entity, Parent +from . import sample_cache from .obfuscate import compile_rules, obfuscate from .pii import PII_REDACTED, pii_column_names from .preserve import read_manual_block @@ -43,8 +44,16 @@ def render_and_write_all( custom_entities: list["TableCatalog"], executor: "Executor | None", config: "Config", + *, + refresh_samples: bool | set[str] = False, ) -> int: - """Render every entity + the index into `config.output_dir`.""" + """Render every entity + the index into `config.output_dir`. + + `refresh_samples` controls the sample-row cache: + - `False` (default): reuse cache when valid, refetch otherwise. + - `True`: ignore cache for every entity, refetch all. + - `set[str]` of entity slugs: refetch just those, reuse cache for the rest. + """ output_dir = config.output_dir output_dir.mkdir(parents=True, exist_ok=True) @@ -53,12 +62,18 @@ def render_and_write_all( obfuscate_value = functools.partial( obfuscate, rules=compile_rules(config.output.obfuscate) ) + obf_hash = sample_cache.obfuscation_hash(config) + refresh_all = refresh_samples is True + refresh_set: set[str] = ( + set(refresh_samples) if isinstance(refresh_samples, (set, list, tuple)) else set() + ) for entity in entities: - head_rows = _fetch_head(executor, entity, sample_size) if executor else [] - body = _render_entity( - entity, head_rows, inline_map.get(entity.slug), obfuscate_value + rendered_rows = _resolve_sample_rows( + entity, executor, sample_size, config, obfuscate_value, obf_hash, + refresh_all=refresh_all, refresh_set=refresh_set, ) + body = _render_entity(entity, rendered_rows, inline_map.get(entity.slug)) _write_file(output_dir / entity.filename, body) index_path = output_dir / "_index.md" @@ -72,9 +87,8 @@ def render_and_write_all( def _render_entity( entity: Entity, - head_rows: list[dict], + rendered_rows: list[dict], custom_entity: "TableCatalog | None", - obfuscate_value: Callable[[str], str], ) -> str: kind_label = "model" if entity.kind == "model" else "source" if custom_entity is not None: @@ -88,9 +102,7 @@ def _render_entity( if custom_entity is not None: lines.extend(custom_entity.render()) lines.extend(_render_parents(entity.parents)) - lines.extend( - _render_sample_rows(head_rows, pii_column_names(entity.columns), obfuscate_value) - ) + lines.extend(_render_sample_rows(rendered_rows)) lines.extend(_render_compiled_sql(entity.compiled_sql)) return "\n".join(lines).rstrip() + "\n" @@ -125,22 +137,19 @@ def _render_parents(parents: list[Parent]) -> list[str]: return lines -def _render_sample_rows( - head_rows: list[dict], - pii_columns: set[str], - obfuscate_value: Callable[[str], str], -) -> list[str]: - """Column-first (transposed) rendering: one bullet per column, pipe-separated values.""" - if not head_rows: +def _render_sample_rows(rendered_rows: list[dict]) -> list[str]: + """Column-first (transposed) rendering: one bullet per column, pipe-separated values. + + Cells are expected to be already obfuscated + truncated (see + `_eagerly_render_rows`). + """ + if not rendered_rows: return [] - cols = list(head_rows[0].keys()) - n_rows = len(head_rows) + cols = list(rendered_rows[0].keys()) + n_rows = len(rendered_rows) lines = [f"## Sample rows ({n_rows} rows, one line per column)", ""] for c in cols: - if c in pii_columns: - values = " | ".join([PII_REDACTED] * n_rows) - else: - values = " | ".join(_cell(row.get(c), obfuscate_value) for row in head_rows) + values = " | ".join(row.get(c, "") for row in rendered_rows) lines.append(f"- `{c}`: {values}") lines.append("") return lines @@ -175,7 +184,62 @@ def _render_index(entities: list[Entity]) -> str: return "\n".join(lines).rstrip() + "\n" -# ─── Sample-row fetching ──────────────────────────────────────────────────── +# ─── Sample-row fetching + caching ────────────────────────────────────────── + + +def _resolve_sample_rows( + entity: Entity, + executor: "Executor | None", + sample_size: int, + cfg: "Config", + obfuscate_value: Callable[[str], str], + obf_hash: str, + *, + refresh_all: bool, + refresh_set: set[str], +) -> list[dict]: + """Return rendered (post-obfuscation) rows for an entity, using cache when possible.""" + if sample_size <= 0: + return [] + + force_refresh = refresh_all or entity.slug in refresh_set + if not force_refresh: + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + return cached + + if executor is None: + # No way to fetch fresh. Fall back to whatever cache exists, even if + # the user asked for refresh — they can't have it without a DB. + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + if force_refresh: + logger.warning( + f" refresh requested for {entity.slug} but no executor configured; " + "reusing cached samples" + ) + return cached + return [] + + raw = _fetch_head(executor, entity, sample_size) + if not raw: + # Fetch failed or table is empty. Don't drop the rendered sample-rows + # section to `[]` if we already have a valid cache — that would flip + # the .md between "with rows" and "without rows" across runs whenever + # a query is transiently flaky. + cached = sample_cache.load(entity, cfg, obf_hash) + if cached is not None: + logger.warning( + f" sample fetch for {entity.slug} returned no rows; reusing cached samples" + ) + return cached + return [] + rendered = _eagerly_render_rows(raw, entity, obfuscate_value) + try: + sample_cache.save(entity, rendered, cfg, obf_hash) + except OSError as e: + logger.warning(f" could not write sample cache for {entity.slug}: {e!r}") + return rendered def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[dict]: @@ -183,7 +247,7 @@ def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[ return [] try: headers, rows, _ = executor.run_direct( - f'select * from {entity.schema_name}.{entity.table_name} limit {sample_size}' + f"select * from {entity.schema_name}.{entity.table_name} limit {sample_size}" ) except Exception as e: logger.warning(f" could not fetch samples for {entity.slug}: {e!r}") @@ -194,6 +258,25 @@ def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[ ] +def _eagerly_render_rows( + head_rows: list[dict], + entity: Entity, + obfuscate_value: Callable[[str], str], +) -> list[dict]: + """Convert raw fetched rows into pre-obfuscated, cell-truncated dicts.""" + pii_cols = pii_column_names(entity.columns) + rendered: list[dict] = [] + for row in head_rows: + rendered_row: dict = {} + for col in row.keys(): + if col in pii_cols: + rendered_row[col] = PII_REDACTED + else: + rendered_row[col] = _cell(row.get(col), obfuscate_value) + rendered.append(rendered_row) + return rendered + + # ─── File writing + manual-notes preservation ─────────────────────────────── diff --git a/src/wlens/render/sample_cache.py b/src/wlens/render/sample_cache.py new file mode 100644 index 0000000..3e1d604 --- /dev/null +++ b/src/wlens/render/sample_cache.py @@ -0,0 +1,125 @@ +"""On-disk cache for sample rows rendered into per-table markdown. + +Goal: skip the warehouse roundtrip on subsequent `wlens generate` runs when +nothing schema-relevant has changed, and pin the rendered rows so they don't +churn the git diff between runs. + +Cache files live under `/..json` (default +`wlens/.cache/samples/`). Each file stores the **post-obfuscation, +post-truncation** cell strings — exactly what gets rendered into the .md — +so the cache itself adds zero new PII surface vs. the markdown that's +already committed. + +Cache invalidates when any of these change for an entity: + +- Set of column names +- Any column's data_type +- Any column's PII flag (added/removed `pii: true` in the dbt schema) +- The effective obfuscation ruleset (defaults + `output.obfuscate` in wlens.yml) +- The cache format `version` +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +from pathlib import Path +from typing import TYPE_CHECKING + +from .obfuscate import compile_rules +from .pii import pii_column_names + +if TYPE_CHECKING: + from ..adapters.base import Entity + from ..config import Config + +logger = logging.getLogger(__name__) + +CACHE_VERSION = 1 +_DOC = ( + "wlens-managed sample cache. Schema docs live in wlens/schema/. " + "Do not read this file — read the .md instead." +) + + +def cache_path(entity: "Entity", cfg: "Config") -> Path: + return cfg.cache_dir / f"{entity.slug}.json" + + +def obfuscation_hash(cfg: "Config") -> str: + """sha256 over the effective scrubber config (defaults + user extras).""" + rules = compile_rules(cfg.output.obfuscate) + payload = [(r.name, r.pattern.pattern, r.replacement) for r in rules] + return "sha256:" + hashlib.sha256( + json.dumps(payload, sort_keys=True).encode() + ).hexdigest() + + +def load(entity: "Entity", cfg: "Config", obf_hash: str) -> list[dict] | None: + """Return cached rendered rows if the cache is valid; otherwise None.""" + path = cache_path(entity, cfg) + if not path.exists(): + return None + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError) as e: + logger.warning(f" sample cache for {entity.slug} unreadable ({e}); refetching") + return None + if data.get("version") != CACHE_VERSION: + return None + if data.get("obfuscation_hash") != obf_hash: + return None + if _columns_from_cache(data.get("columns") or []) != _columns_view(entity): + return None + rows = data.get("rows") + if not isinstance(rows, list): + return None + return rows + + +def save( + entity: "Entity", + rendered_rows: list[dict], + cfg: "Config", + obf_hash: str, +) -> None: + """Write rendered (post-obfuscation) rows to the cache.""" + path = cache_path(entity, cfg) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "_doc": _DOC, + "version": CACHE_VERSION, + "schema": entity.schema_name, + "table": entity.table_name, + "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "obfuscation_hash": obf_hash, + "columns": _columns_for_write(entity), + "rows": rendered_rows, + } + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _columns_for_write(entity: "Entity") -> list[dict]: + pii = pii_column_names(entity.columns) + return [ + {"name": c.name, "data_type": c.data_type, "pii": c.name in pii} + for c in entity.columns.values() + ] + + +def _columns_view(entity: "Entity") -> dict: + pii = pii_column_names(entity.columns) + return { + c.name: {"data_type": c.data_type, "pii": c.name in pii} + for c in entity.columns.values() + } + + +def _columns_from_cache(cached: list[dict]) -> dict: + return { + c["name"]: {"data_type": c.get("data_type"), "pii": bool(c.get("pii", False))} + for c in cached + if isinstance(c, dict) and "name" in c + } diff --git a/tests/test_cli.py b/tests/test_cli.py index c6613b6..c3d7039 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -60,12 +60,62 @@ def test_init_writes_wlens_gitignore(tmp_path: Path, monkeypatch): gi = tmp_path / "wlens" / ".gitignore" assert gi.exists() body = gi.read_text() - # cache/ and share/ must be ignored — schema/ must NOT be (it's the point). - assert "cache/" in body + # SQL query cache + share/ ignored; sample-row cache explicitly tracked; + # generated schema markdown stays out of the ignore list (it's the point). + assert ".cache/*" in body + assert "!.cache/samples/" in body assert "share/" in body assert "schema/" not in body +def test_generate_migrates_legacy_wlens_gitignore(tmp_path: Path, monkeypatch, dbt_project): + """An older `wlens/.gitignore` (pre-`.cache/` rename) is rewritten in place.""" + import textwrap + from wlens.cli import WLENS_GITIGNORE_BODY, WLENS_GITIGNORE_LEGACY + + monkeypatch.chdir(tmp_path) + (tmp_path / "wlens.yml").write_text( + textwrap.dedent(f""" + adapter: + kind: dbt + project_dir: {dbt_project.relative_to(tmp_path)} + output: + dir: wlens/schema + include_sample_rows: false + """).lstrip() + ) + gi = tmp_path / "wlens" / ".gitignore" + gi.parent.mkdir(parents=True, exist_ok=True) + gi.write_text(WLENS_GITIGNORE_LEGACY) + + assert main(["generate"]) == 0 + assert gi.read_text() == WLENS_GITIGNORE_BODY + + +def test_generate_leaves_custom_wlens_gitignore_alone(tmp_path: Path, monkeypatch, dbt_project): + """If the user has hand-edited `wlens/.gitignore`, don't touch it.""" + import textwrap + + monkeypatch.chdir(tmp_path) + (tmp_path / "wlens.yml").write_text( + textwrap.dedent(f""" + adapter: + kind: dbt + project_dir: {dbt_project.relative_to(tmp_path)} + output: + dir: wlens/schema + include_sample_rows: false + """).lstrip() + ) + gi = tmp_path / "wlens" / ".gitignore" + gi.parent.mkdir(parents=True, exist_ok=True) + custom = "# my own rules\nfoo/\nbar/\n" + gi.write_text(custom) + + assert main(["generate"]) == 0 + assert gi.read_text() == custom + + def test_init_defaults_project_dir_to_dot_when_no_dbt_project(tmp_path: Path, monkeypatch): monkeypatch.chdir(tmp_path) main(["init"]) diff --git a/tests/test_render_markdown.py b/tests/test_render_markdown.py index b7ed2e2..aa35978 100644 --- a/tests/test_render_markdown.py +++ b/tests/test_render_markdown.py @@ -35,8 +35,10 @@ class _StubExecutor: def __init__(self, headers: list[str], rows: list[tuple]): self._headers = headers self._rows = rows + self.calls = 0 def run_direct(self, sql: str): # noqa: ARG002 — sql ignored, fixed payload + self.calls += 1 return self._headers, self._rows, False @@ -343,3 +345,149 @@ def test_plans_example_renders(): # auto-render still handles plain scalars. assert "**Price:** $0" in md assert "**Seats included:** 5" in md + + +# ─── Sample-row cache ─────────────────────────────────────────────────────── + + +def _widget_executor() -> _StubExecutor: + return _StubExecutor( + headers=["widget_id", "color"], + rows=[(1, "red"), (2, "blue")], + ) + + +def _widget_only(entities): + return [e for e in entities if e.slug == "prod.dim_widget"] + + +def test_sample_cache_skips_requery_on_second_run(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + first_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert executor.calls == 1 + cache_file = cfg.cache_dir / "prod.dim_widget.json" + assert cache_file.exists() + + # Second run: same executor, same cache → zero new queries, identical output. + render_and_write_all(entities, [], executor, cfg) + second_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert executor.calls == 1, "cache hit must not re-query" + assert first_md == second_md + + +def test_sample_cache_refetches_when_column_drifts(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + # Mutate a column's data_type — cache should invalidate just for this entity. + next(iter(entities[0].columns.values())).data_type = "TOTALLY_NEW_TYPE" + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 2 + + +def test_sample_cache_refetches_when_obfuscation_changes(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + cfg.output.obfuscate = [{"pattern": "red", "replacement": ""}] + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 2 + md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "" in md + + +def test_refresh_samples_all_forces_requery(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + assert executor.calls == 1 + + render_and_write_all(entities, [], executor, cfg, refresh_samples=True) + assert executor.calls == 2 + + +def test_refresh_samples_slug_targets_one_entity(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = [ + e for e in DbtAdapter(cfg).list_entities() + if e.slug in {"prod.dim_widget", "prod.fct_sale"} + ] + executor = _StubExecutor(headers=["a"], rows=[(1,)]) + + render_and_write_all(entities, [], executor, cfg) + base_calls = executor.calls + assert base_calls >= 2 # one per entity on cold cache + + render_and_write_all( + entities, [], executor, cfg, refresh_samples={"prod.dim_widget"} + ) + # Only one entity refetched; the other should hit the cache. + assert executor.calls == base_calls + 1 + + +class _FlakyExecutor(_StubExecutor): + """Returns headers/rows on the first call, then empty on subsequent calls.""" + + def run_direct(self, sql: str): # noqa: ARG002 + self.calls += 1 + if self.calls == 1: + return self._headers, self._rows, False + return self._headers, [], False + + +def test_refresh_falls_back_to_cache_when_fetch_returns_empty(tmp_path, dbt_project): + """Forced refresh that comes back empty must not drop the cache or the + rendered section — the next run should still produce identical .md.""" + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + + executor = _FlakyExecutor(headers=["widget_id", "color"], rows=[(1, "red"), (2, "blue")]) + + render_and_write_all(entities, [], executor, cfg) + first_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "## Sample rows" in first_md + + # Second run: --refresh-samples forces a fetch, but the executor returns + # nothing. The .md must still contain the previously-cached rows. + render_and_write_all(entities, [], executor, cfg, refresh_samples=True) + second_md = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert second_md == first_md + assert executor.calls == 2 # the refresh did try to fetch + + +def test_sample_cache_reused_when_no_executor(tmp_path, dbt_project): + cfg = load_config(_setup_project(tmp_path, dbt_project)) + cfg.output.include_sample_rows = True + entities = _widget_only(DbtAdapter(cfg).list_entities()) + executor = _widget_executor() + + render_and_write_all(entities, [], executor, cfg) + md_with_executor = (cfg.output_dir / "prod.dim_widget.md").read_text() + + # Wipe the markdown so we know the second render had to build it from scratch. + (cfg.output_dir / "prod.dim_widget.md").unlink() + + render_and_write_all(entities, [], None, cfg) + md_offline = (cfg.output_dir / "prod.dim_widget.md").read_text() + assert "## Sample rows" in md_offline + assert md_offline == md_with_executor diff --git a/uv.lock b/uv.lock index f81a924..b9b1cda 100644 --- a/uv.lock +++ b/uv.lock @@ -1159,7 +1159,7 @@ wheels = [ [[package]] name = "wlens" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "anyio" },