cache sample rows on disk

.gitignore

@@ -22,3 +22,4 @@ htmlcov/
 .vscode/
 
 .wlens-cache/
+wlens/.cache/sql/

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "wlens"
-version = "0.2.0"
+version = "0.3.0"
 description = "Warehouse lens for AI agents — per-table markdown docs + read-only SQL adaptor, zero-runtime schema exploration for Claude Code, Cursor, Continue."
 readme = "README.md"
 license = { file = "LICENSE" }

src/wlens/cli.py

@@ -24,6 +24,16 @@
 
 logger = logging.getLogger(__name__)
 
+WLENS_GITIGNORE_BODY = (
+    "# wlens-managed — do not edit.\n"
+    ".cache/*\n"
+    "!.cache/samples/\n"
+    "share/\n"
+)
+# Predecessor body (pre-`.cache/` rename). `wlens generate` rewrites the file
+# in-place when it sees this exact content so existing installs migrate.
+WLENS_GITIGNORE_LEGACY = "# wlens-managed — do not edit.\ncache/\nshare/\n"
+
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
@@ -39,6 +49,17 @@ def main(argv: list[str] | None = None) -> int:
     p_gen.add_argument("--config", help=f"Path to {DEFAULT_CONFIG_FILENAME} (default: walk up from cwd).")
     p_gen.add_argument("--output-dir", help="Override output directory from config.")
     p_gen.add_argument("--skip-samples", action="store_true", help="Don't fetch sample rows from the warehouse.")
+    p_gen.add_argument(
+        "--refresh-samples",
+        nargs="?",
+        const="*",
+        default=None,
+        metavar="SLUG[,SLUG…]",
+        help=(
+            "Force-refresh sample rows. Pass with no value to refresh every table, "
+            "or with a comma-separated list of `schema.table` slugs to refresh just those."
+        ),
+    )
 
     p_query = sub.add_parser("query", help="Execute a read-only SQL query.")
     p_query.add_argument("sql", nargs="?", help="SQL to execute (or piped via stdin).")
@@ -124,6 +145,7 @@ def main(argv: list[str] | None = None) -> int:
             config_path=Path(args.config) if args.config else None,
             output_override=args.output_dir,
             skip_samples=args.skip_samples,
+            refresh_samples=args.refresh_samples,
         )
     if args.command == "query":
         return _cmd_query(
@@ -285,12 +307,13 @@ def _cmd_init(*, force: bool) -> int:
         dest.write_text(content)
         logger.info(f"wrote {dest.relative_to(cwd)}")
 
-    # Seed `wlens/.gitignore` so cache + share stay out of the consumer's repo.
-    # The generated schema markdown IS meant to be committed — not ignored.
+    # Seed `wlens/.gitignore`. Schema markdown + sample cache are committed
+    # (the latter pins sample rows for stable diffs); SQL query cache and
+    # share/ stay out of the consumer's repo.
     gitignore = cwd / "wlens" / ".gitignore"
     if not gitignore.exists() or force:
         gitignore.parent.mkdir(parents=True, exist_ok=True)
-        gitignore.write_text("# wlens-managed — do not edit.\ncache/\nshare/\n")
+        gitignore.write_text(WLENS_GITIGNORE_BODY)
         logger.info(f"wrote {gitignore.relative_to(cwd)}")
 
     logger.info("")
@@ -390,7 +413,28 @@ def _set_yaml_value(yaml_content: str, key: str, value: str) -> str:
     return "".join(out_lines)
 
 
-def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip_samples: bool) -> int:
+def _migrate_wlens_gitignore(repo_root: Path) -> None:
+    """Rewrite a pre-`.cache/` `wlens/.gitignore` to the new pattern in place.
+
+    One-shot migration helper. Safe to delete (along with `WLENS_GITIGNORE_LEGACY`
+    and the two test cases in tests/test_cli.py) once existing installs have all
+    run `wlens generate` at least once after the rename — target 0.5+.
+    """
+    path = repo_root / "wlens" / ".gitignore"
+    if not path.exists():
+        return
+    if path.read_text() == WLENS_GITIGNORE_LEGACY:
+        path.write_text(WLENS_GITIGNORE_BODY)
+        logger.info(f"updated {path.relative_to(repo_root)} for new cache layout")
+
+
+def _cmd_generate(
+    *,
+    config_path: Path | None,
+    output_override: str | None,
+    skip_samples: bool,
+    refresh_samples: str | None,
+) -> int:
     from .adapters.dbt import DbtAdapter
     from .entities.loader import load_entities
     from .executor import build_executor
@@ -400,6 +444,8 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip
     if output_override:
         config.output.dir = output_override
 
+    _migrate_wlens_gitignore(config.repo_root)
+
     if config.adapter.kind != "dbt":
         raise NotImplementedError(
             f"Adapter kind {config.adapter.kind!r} not yet implemented. "
@@ -419,7 +465,15 @@ def _cmd_generate(*, config_path: Path | None, output_override: str | None, skip
             logger.warning(f"could not build executor ({e!r}) — continuing without sample rows.")
             executor = None
 
-    count = render_and_write_all(entities, custom_entities, executor, config)
+    refresh_arg: bool | set[str] = False
+    if refresh_samples == "*":
+        refresh_arg = True
+    elif refresh_samples:
+        refresh_arg = {s.strip() for s in refresh_samples.split(",") if s.strip()}
+
+    count = render_and_write_all(
+        entities, custom_entities, executor, config, refresh_samples=refresh_arg
+    )
     logger.info(f"wrote {count} entity files to {config.output.dir}")
     if executor is not None:
         executor.close()
@@ -504,7 +558,7 @@ def _cmd_clean(
     ]
     # Back-compat: previous versions scattered files at the repo root. Clean
     # them up if they're still there.
-    for legacy in (".wlens-cache", "wlens-share", ".claude/schema"):
+    for legacy in (".wlens-cache", "wlens/cache", "wlens-share", ".claude/schema"):
         legacy_path = repo_root / legacy
         if legacy_path.exists():
             targets.append(legacy_path)

src/wlens/config.py

@@ -47,6 +47,7 @@
 
 DEFAULT_CONFIG_FILENAME = "wlens.yml"
 DEFAULT_OUTPUT_DIR = "wlens/schema"
+DEFAULT_CACHE_DIR = "wlens/.cache/samples"
 DEFAULT_SAMPLE_SIZE = 5
 ENV_VAR_PATTERN = re.compile(r"\$\{([A-Z_][A-Z0-9_]*)\}")
 
@@ -75,6 +76,7 @@ class ExecutorConfig:
 @dataclass
 class OutputConfig:
     dir: str = DEFAULT_OUTPUT_DIR
+    cache_dir: str = DEFAULT_CACHE_DIR
     include_sample_rows: bool = True
     sample_size: int = DEFAULT_SAMPLE_SIZE
     # Extra value-level obfuscation rules appended to the built-in defaults
@@ -103,6 +105,10 @@ class Config:
     def output_dir(self) -> Path:
         return (self.repo_root / self.output.dir).resolve()
 
+    @property
+    def cache_dir(self) -> Path:
+        return (self.repo_root / self.output.cache_dir).resolve()
+
 
 def find_config(start: Path | None = None) -> Path:
     """Walk up from `start` looking for a wlens.yml."""
@@ -160,6 +166,7 @@ def _build_config(raw: dict, repo_root: Path) -> Config:
     )
     output = OutputConfig(
         dir=output_raw.get("dir", DEFAULT_OUTPUT_DIR),
+        cache_dir=output_raw.get("cache_dir", DEFAULT_CACHE_DIR),
         include_sample_rows=bool(output_raw.get("include_sample_rows", True)),
         sample_size=int(output_raw.get("sample_size", DEFAULT_SAMPLE_SIZE)),
         obfuscate=list(output_raw.get("obfuscate") or []),

src/wlens/executor/base.py

@@ -25,7 +25,7 @@
 
 logger = logging.getLogger(__name__)
 
-CACHE_DIRNAME = "wlens/cache"
+CACHE_DIRNAME = "wlens/.cache"
 CACHE_TTL_SECONDS = 24 * 60 * 60  # 24 hours
 CACHE_SUBDIR = "sql"
 
@@ -110,7 +110,7 @@ def run(
     ) -> tuple[list[str], list[tuple], bool]:
         """Execute a guarded query, return (headers, rows, cache_hit).
 
-        Caches to `.wlens-cache/sql/` with a daily TTL.
+        Caches to `wlens/.cache/sql/` with a daily TTL.
         """
         assert_read_only(sql)
         if use_cache:

src/wlens/render/markdown.py

@@ -20,6 +20,7 @@
 from typing import TYPE_CHECKING, Callable
 
 from ..adapters.base import Column, Entity, Parent
+from . import sample_cache
 from .obfuscate import compile_rules, obfuscate
 from .pii import PII_REDACTED, pii_column_names
 from .preserve import read_manual_block
@@ -43,8 +44,16 @@ def render_and_write_all(
     custom_entities: list["TableCatalog"],
     executor: "Executor | None",
     config: "Config",
+    *,
+    refresh_samples: bool | set[str] = False,
 ) -> int:
-    """Render every entity + the index into `config.output_dir`."""
+    """Render every entity + the index into `config.output_dir`.
+
+    `refresh_samples` controls the sample-row cache:
+    - `False` (default): reuse cache when valid, refetch otherwise.
+    - `True`: ignore cache for every entity, refetch all.
+    - `set[str]` of entity slugs: refetch just those, reuse cache for the rest.
+    """
     output_dir = config.output_dir
     output_dir.mkdir(parents=True, exist_ok=True)
 
@@ -53,12 +62,18 @@ def render_and_write_all(
     obfuscate_value = functools.partial(
         obfuscate, rules=compile_rules(config.output.obfuscate)
     )
+    obf_hash = sample_cache.obfuscation_hash(config)
+    refresh_all = refresh_samples is True
+    refresh_set: set[str] = (
+        set(refresh_samples) if isinstance(refresh_samples, (set, list, tuple)) else set()
+    )
 
     for entity in entities:
-        head_rows = _fetch_head(executor, entity, sample_size) if executor else []
-        body = _render_entity(
-            entity, head_rows, inline_map.get(entity.slug), obfuscate_value
+        rendered_rows = _resolve_sample_rows(
+            entity, executor, sample_size, config, obfuscate_value, obf_hash,
+            refresh_all=refresh_all, refresh_set=refresh_set,
         )
+        body = _render_entity(entity, rendered_rows, inline_map.get(entity.slug))
         _write_file(output_dir / entity.filename, body)
 
     index_path = output_dir / "_index.md"
@@ -72,9 +87,8 @@ def render_and_write_all(
 
 def _render_entity(
     entity: Entity,
-    head_rows: list[dict],
+    rendered_rows: list[dict],
     custom_entity: "TableCatalog | None",
-    obfuscate_value: Callable[[str], str],
 ) -> str:
     kind_label = "model" if entity.kind == "model" else "source"
     if custom_entity is not None:
@@ -88,9 +102,7 @@ def _render_entity(
     if custom_entity is not None:
         lines.extend(custom_entity.render())
     lines.extend(_render_parents(entity.parents))
-    lines.extend(
-        _render_sample_rows(head_rows, pii_column_names(entity.columns), obfuscate_value)
-    )
+    lines.extend(_render_sample_rows(rendered_rows))
     lines.extend(_render_compiled_sql(entity.compiled_sql))
     return "\n".join(lines).rstrip() + "\n"
 
@@ -125,22 +137,19 @@ def _render_parents(parents: list[Parent]) -> list[str]:
     return lines
 
 
-def _render_sample_rows(
-    head_rows: list[dict],
-    pii_columns: set[str],
-    obfuscate_value: Callable[[str], str],
-) -> list[str]:
-    """Column-first (transposed) rendering: one bullet per column, pipe-separated values."""
-    if not head_rows:
+def _render_sample_rows(rendered_rows: list[dict]) -> list[str]:
+    """Column-first (transposed) rendering: one bullet per column, pipe-separated values.
+
+    Cells are expected to be already obfuscated + truncated (see
+    `_eagerly_render_rows`).
+    """
+    if not rendered_rows:
         return []
-    cols = list(head_rows[0].keys())
-    n_rows = len(head_rows)
+    cols = list(rendered_rows[0].keys())
+    n_rows = len(rendered_rows)
     lines = [f"## Sample rows ({n_rows} rows, one line per column)", ""]
     for c in cols:
-        if c in pii_columns:
-            values = " | ".join([PII_REDACTED] * n_rows)
-        else:
-            values = " | ".join(_cell(row.get(c), obfuscate_value) for row in head_rows)
+        values = " | ".join(row.get(c, "") for row in rendered_rows)
         lines.append(f"- `{c}`: {values}")
     lines.append("")
     return lines
@@ -175,15 +184,70 @@ def _render_index(entities: list[Entity]) -> str:
     return "\n".join(lines).rstrip() + "\n"
 
 
-# ─── Sample-row fetching ────────────────────────────────────────────────────
+# ─── Sample-row fetching + caching ──────────────────────────────────────────
+
+
+def _resolve_sample_rows(
+    entity: Entity,
+    executor: "Executor | None",
+    sample_size: int,
+    cfg: "Config",
+    obfuscate_value: Callable[[str], str],
+    obf_hash: str,
+    *,
+    refresh_all: bool,
+    refresh_set: set[str],
+) -> list[dict]:
+    """Return rendered (post-obfuscation) rows for an entity, using cache when possible."""
+    if sample_size <= 0:
+        return []
+
+    force_refresh = refresh_all or entity.slug in refresh_set
+    if not force_refresh:
+        cached = sample_cache.load(entity, cfg, obf_hash)
+        if cached is not None:
+            return cached
+
+    if executor is None:
+        # No way to fetch fresh. Fall back to whatever cache exists, even if
+        # the user asked for refresh — they can't have it without a DB.
+        cached = sample_cache.load(entity, cfg, obf_hash)
+        if cached is not None:
+            if force_refresh:
+                logger.warning(
+                    f"  refresh requested for {entity.slug} but no executor configured; "
+                    "reusing cached samples"
+                )
+            return cached
+        return []
+
+    raw = _fetch_head(executor, entity, sample_size)
+    if not raw:
+        # Fetch failed or table is empty. Don't drop the rendered sample-rows
+        # section to `[]` if we already have a valid cache — that would flip
+        # the .md between "with rows" and "without rows" across runs whenever
+        # a query is transiently flaky.
+        cached = sample_cache.load(entity, cfg, obf_hash)
+        if cached is not None:
+            logger.warning(
+                f"  sample fetch for {entity.slug} returned no rows; reusing cached samples"
+            )
+            return cached
+        return []
+    rendered = _eagerly_render_rows(raw, entity, obfuscate_value)
+    try:
+        sample_cache.save(entity, rendered, cfg, obf_hash)
+    except OSError as e:
+        logger.warning(f"  could not write sample cache for {entity.slug}: {e!r}")
+    return rendered
 
 
 def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[dict]:
     if sample_size <= 0:
         return []
     try:
         headers, rows, _ = executor.run_direct(
-            f'select * from {entity.schema_name}.{entity.table_name} limit {sample_size}'
+            f"select * from {entity.schema_name}.{entity.table_name} limit {sample_size}"
         )
     except Exception as e:
         logger.warning(f"  could not fetch samples for {entity.slug}: {e!r}")
@@ -194,6 +258,25 @@ def _fetch_head(executor: "Executor", entity: Entity, sample_size: int) -> list[
     ]
 
 
+def _eagerly_render_rows(
+    head_rows: list[dict],
+    entity: Entity,
+    obfuscate_value: Callable[[str], str],
+) -> list[dict]:
+    """Convert raw fetched rows into pre-obfuscated, cell-truncated dicts."""
+    pii_cols = pii_column_names(entity.columns)
+    rendered: list[dict] = []
+    for row in head_rows:
+        rendered_row: dict = {}
+        for col in row.keys():
+            if col in pii_cols:
+                rendered_row[col] = PII_REDACTED
+            else:
+                rendered_row[col] = _cell(row.get(col), obfuscate_value)
+        rendered.append(rendered_row)
+    return rendered
+
+
 # ─── File writing + manual-notes preservation ───────────────────────────────