From 326e780f0bee22e379a58720e04d77c1d7458908 Mon Sep 17 00:00:00 2001 From: David Baussart Date: Wed, 2 Sep 2026 08:14:33 +0200 Subject: [PATCH] feat(executor): add credential process --- README.md | 4 +- docs/quickstart.md | 49 +++++++++ docs/wlens-yml-reference.md | 42 ++++++++ src/wlens/config.py | 20 ++++ src/wlens/executor/__init__.py | 6 +- src/wlens/executor/credentials.py | 85 ++++++++++++++++ src/wlens/templates/wlens.yml | 2 + tests/test_executor_credentials.py | 157 +++++++++++++++++++++++++++++ 8 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 src/wlens/executor/credentials.py create mode 100644 tests/test_executor_credentials.py diff --git a/README.md b/README.md index 78e8a4f..8e92f33 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,9 @@ line. The markdown is committed to your repo. The agent can grep and read it without warehouse credentials. Credentials are only needed when a query -runs. +runs. They can come from environment variables or a lazy +[`credential_process`](docs/wlens-yml-reference.md#credential-process), which +keeps them out of the agent's ambient environment. ### Why markdown diff --git a/docs/quickstart.md b/docs/quickstart.md index fb4cada..96e5584 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -57,12 +57,61 @@ executor: # database: ${WLENS_DB_NAME} # user: ${WLENS_DB_USER} # password: ${WLENS_DB_PASSWORD} + # Or load those fields lazily from JSON instead of exporting them: + # credential_process: [./scripts/wlens-credentials] ``` Env-var substitution (`${SOME_VAR}`) happens at load time, so secrets stay out of the committed file. Set them however you prefer (`.env`, direnv, your platform's secret manager). +To avoid putting warehouse credentials in the caller's environment, configure +a process that returns the connection fields as JSON instead: + +```yaml +executor: + kind: redshift + credential_process: [./scripts/wlens-credentials] +``` + +The command must write exactly one JSON object to stdout and nothing else: + +```json +{ + "host": "warehouse.example.com", + "port": 5439, + "database": "analytics", + "user": "reader", + "password": "secret" +} +``` + +It may return any subset of `host`, `port`, `database`, `user`, `password`, and +`path`. Returned values override the corresponding fields written directly in +`wlens.yml`, so a common setup keeps `host`, `port`, and `database` static while +the process returns only `user` and `password`. + +The process can be any executable that produces that JSON. For example, if an +AWS Secrets Manager secret named `analytics/wlens` contains the object above: + +```yaml +credential_process: + - aws + - secretsmanager + - get-secret-value + - --secret-id + - analytics/wlens + - --query + - SecretString + - --output + - text +``` + +The command runs directly without a shell, relative to the directory containing +`wlens.yml`, and only when wlens needs an executor. See the +[`wlens.yml` reference](wlens-yml-reference.md#credential-process) for complete +validation and security behavior. + ## 4. Compile dbt wlens reads `/target/manifest.json`, so make sure it's fresh. diff --git a/docs/wlens-yml-reference.md b/docs/wlens-yml-reference.md index 95c0480..a4f2ee8 100644 --- a/docs/wlens-yml-reference.md +++ b/docs/wlens-yml-reference.md @@ -50,12 +50,54 @@ queries entirely — you'll still get schema docs. | `database` | — | Server engines only. | | `user` | — | Server engines only. | | `password` | — | Server engines only. | +| `credential_process` | — | Command that returns connection fields as JSON. Accepts an executable string or an argument list. | **Environment-variable expansion.** Any string value of the form `${SOME_NAME}` is expanded from the process environment when `wlens.yml` is loaded. Missing variables become empty strings. This keeps secrets out of the committed file. +### Credential process + +Use `credential_process` when credentials should be loaded only for warehouse +operations instead of exported into the shell running wlens: + +```yaml +executor: + kind: redshift + credential_process: + - ./scripts/wlens-credentials +``` + +The command must write one JSON object to stdout. It may return any subset of +`host`, `port`, `database`, `user`, `password`, and `path`; returned values +override the corresponding static executor fields. + +```json +{ + "host": "warehouse.example.com", + "port": 5439, + "database": "analytics", + "user": "reader", + "password": "..." +} +``` + +The process runs only when wlens builds an executor for a query, sample-row +fetch, or MCP server. Relative executable paths and arguments resolve from the +directory containing `wlens.yml`. Wlens captures the output in memory and does +not add the returned values to its environment or the caller's environment. + +Commands run directly, without a shell. Use a YAML list when passing arguments: + +```yaml +credential_process: [./scripts/wlens-credentials, --profile, analytics] +``` + +Do not put secrets directly in command arguments because other local processes +may be able to inspect process arguments. Stdout must contain only the JSON +object. For safety, wlens does not reproduce stdout or stderr in errors. + ### DuckDB ```yaml diff --git a/src/wlens/config.py b/src/wlens/config.py index 0033971..797edb1 100644 --- a/src/wlens/config.py +++ b/src/wlens/config.py @@ -17,6 +17,8 @@ database: ${WLENS_DB_NAME} user: ${WLENS_DB_USER} password: ${WLENS_DB_PASSWORD} + # Or load connection fields from a subprocess when the executor is built: + # credential_process: [./scripts/wlens-credentials] output: dir: .claude/schema @@ -69,6 +71,7 @@ class ExecutorConfig: database: str | None = None user: str | None = None password: str | None = None + credential_process: list[str] = field(default_factory=list) # File path (or `:memory:`) for file-based engines like DuckDB. path: str | None = None @@ -162,6 +165,7 @@ def _build_config(raw: dict, repo_root: Path) -> Config: database=_str_or_none(executor_raw.get("database")), user=_str_or_none(executor_raw.get("user")), password=_str_or_none(executor_raw.get("password")), + credential_process=_parse_credential_process(executor_raw.get("credential_process")), path=_str_or_none(executor_raw.get("path")), ) output = OutputConfig( @@ -199,6 +203,22 @@ def _build_config(raw: dict, repo_root: Path) -> Config: ) +def _parse_credential_process(value: Any) -> list[str]: + if isinstance(value, str) and value.strip(): + return [value] + if ( + isinstance(value, list) + and value + and all(isinstance(argument, str) and argument for argument in value) + ): + return list(value) + if value is None: + return [] + raise ValueError( + "`executor.credential_process` must be a command string or a non-empty list of strings." + ) + + def _str_or_none(value: Any) -> str | None: if value is None: return None diff --git a/src/wlens/executor/__init__.py b/src/wlens/executor/__init__.py index 521a0d6..95d1065 100644 --- a/src/wlens/executor/__init__.py +++ b/src/wlens/executor/__init__.py @@ -12,9 +12,13 @@ __all__ = ["Executor", "ReadOnlyViolation", "build_executor", "format_markdown_table"] -def build_executor(config: "Config") -> Executor: +def build_executor(config: Config) -> Executor: """Instantiate the executor indicated by `wlens.yml`.""" + from .credentials import resolve_credentials + kind = (config.executor.kind or "").lower() + if kind in {"redshift", "postgres", "postgresql", "duckdb"}: + config = resolve_credentials(config) if kind in ("redshift",): from .redshift import RedshiftExecutor return RedshiftExecutor(config) diff --git a/src/wlens/executor/credentials.py b/src/wlens/executor/credentials.py new file mode 100644 index 0000000..99d4116 --- /dev/null +++ b/src/wlens/executor/credentials.py @@ -0,0 +1,85 @@ +"""Resolve executor connection fields from an external credential process.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import replace +from typing import Any + +from ..config import Config + +CREDENTIAL_PROCESS_TIMEOUT_SECONDS = 30 +CONNECTION_FIELDS = frozenset({"host", "port", "database", "user", "password", "path"}) + + +def resolve_credentials(config: Config) -> Config: + """Return a config with credential-process output merged into its executor. + + The subprocess runs relative to the directory containing ``wlens.yml``. + Its output is kept in memory and never copied into ``os.environ``. The + original config is not mutated. + """ + command = config.executor.credential_process + if not command: + return config + + try: + completed = subprocess.run( + command, + cwd=config.repo_root, + capture_output=True, + text=True, + timeout=CREDENTIAL_PROCESS_TIMEOUT_SECONDS, + check=False, + ) + except FileNotFoundError: + raise ValueError("executor.credential_process executable was not found") from None + except subprocess.TimeoutExpired: + raise ValueError( + "executor.credential_process timed out after " + f"{CREDENTIAL_PROCESS_TIMEOUT_SECONDS} seconds" + ) from None + except OSError: + raise ValueError("executor.credential_process could not be started") from None + + if completed.returncode != 0: + raise ValueError( + f"executor.credential_process exited with status {completed.returncode}" + ) + + try: + payload = json.loads(completed.stdout) + except (json.JSONDecodeError, TypeError): + raise ValueError("executor.credential_process must return valid JSON") from None + + if not isinstance(payload, dict): + raise ValueError("executor.credential_process must return a JSON object") + + unsupported_fields = sorted(set(payload) - CONNECTION_FIELDS) + if unsupported_fields: + raise ValueError( + "executor.credential_process returned unsupported fields: " + + ", ".join(unsupported_fields) + ) + + updates = {name: _normalize_field(name, value) for name, value in payload.items()} + executor = replace(config.executor, credential_process=[], **updates) + return replace(config, executor=executor) + + +def _normalize_field(name: str, value: Any) -> str | int | None: + if name == "port": + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + raise ValueError( + "executor.credential_process returned an invalid value for port" + ) from None + + if value is None: + return None + string_value = str(value) + return string_value if string_value else None diff --git a/src/wlens/templates/wlens.yml b/src/wlens/templates/wlens.yml index 0fccab8..8f18e6b 100644 --- a/src/wlens/templates/wlens.yml +++ b/src/wlens/templates/wlens.yml @@ -15,6 +15,8 @@ executor: # database: ${WLENS_DB_NAME} # user: ${WLENS_DB_USER} # password: ${WLENS_DB_PASSWORD} + # Or load those fields lazily from JSON instead of exporting them: + # credential_process: [./scripts/wlens-credentials] # Value-level scrubbing of sample-row cells. # Defaults always on: email, IPv4, phone → / / . diff --git a/tests/test_executor_credentials.py b/tests/test_executor_credentials.py new file mode 100644 index 0000000..2f9b9ab --- /dev/null +++ b/tests/test_executor_credentials.py @@ -0,0 +1,157 @@ +"""Executor credential-process loading.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import yaml + +from wlens.config import load_config +from wlens.executor import build_executor + + +def _write_config(tmp_path: Path, executor: dict) -> Path: + config_path = tmp_path / "wlens.yml" + config_path.write_text( + yaml.safe_dump( + { + "adapter": {"kind": "dbt"}, + "executor": {"kind": "redshift", **executor}, + }, + sort_keys=False, + ) + ) + return config_path + + +def _write_process(tmp_path: Path, body: str) -> list[str]: + process_path = tmp_path / "credentials.py" + process_path.write_text(body) + return [sys.executable, process_path.name] + + +def test_credential_process_is_lazy_and_resolves_from_config_directory(tmp_path: Path): + command = _write_process( + tmp_path, + """ +import json +from pathlib import Path + +Path("process-called").write_text("yes") +print(json.dumps({ + "host": "warehouse.example.com", + "port": 5439, + "database": "analytics", + "user": "reader", + "password": "secret", +})) +""", + ) + config = load_config(_write_config(tmp_path, {"credential_process": command})) + + assert config.executor.credential_process == command + assert config.executor.host is None + assert not (tmp_path / "process-called").exists() + + executor = build_executor(config) + + assert (tmp_path / "process-called").read_text() == "yes" + assert executor.config.executor.host == "warehouse.example.com" + assert executor.config.executor.port == 5439 + assert executor.config.executor.database == "analytics" + assert executor.config.executor.user == "reader" + assert executor.config.executor.password == "secret" + assert executor.config.executor.credential_process == [] + assert config.executor.host is None, "resolving credentials must not mutate the loaded config" + + +def test_credential_process_overrides_only_returned_executor_fields(tmp_path: Path): + command = _write_process( + tmp_path, + f"print({json.dumps(json.dumps({'user': 'dynamic-user', 'password': 'dynamic-pass'}))})\n", + ) + config = load_config( + _write_config( + tmp_path, + { + "host": "warehouse.example.com", + "port": 15439, + "database": "analytics", + "user": "static-user", + "credential_process": command, + }, + ) + ) + + executor = build_executor(config) + + assert executor.config.executor.host == "warehouse.example.com" + assert executor.config.executor.port == 15439 + assert executor.config.executor.database == "analytics" + assert executor.config.executor.user == "dynamic-user" + assert executor.config.executor.password == "dynamic-pass" + + +def test_unsupported_executor_does_not_run_credential_process(tmp_path: Path): + command = _write_process( + tmp_path, + 'from pathlib import Path; Path("process-called").write_text("yes")\n', + ) + config = load_config( + _write_config(tmp_path, {"kind": "not-a-warehouse", "credential_process": command}) + ) + + with pytest.raises(ValueError, match="Unsupported executor kind"): + build_executor(config) + + assert not (tmp_path / "process-called").exists() + + +def test_scalar_credential_process_is_normalized_to_one_argument(tmp_path: Path): + config = load_config(_write_config(tmp_path, {"credential_process": "./credentials"})) + + assert config.executor.credential_process == ["./credentials"] + + +@pytest.mark.parametrize( + "value", + [[], 42, ["valid", 42]], +) +def test_invalid_credential_process_config_is_rejected(tmp_path: Path, value): + with pytest.raises(ValueError, match="executor.credential_process"): + load_config(_write_config(tmp_path, {"credential_process": value})) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ( + 'import sys; print("do-not-leak"); print("do-not-leak", file=sys.stderr); ' + "raise SystemExit(7)\n", + "exited with status 7", + ), + ('print("do-not-leak invalid json")\n', "valid JSON"), + ('print(\'["do-not-leak"]\')\n', "JSON object"), + ( + 'print(\'{"unexpected": "do-not-leak"}\')\n', + "unsupported fields: unexpected", + ), + ('print(\'{"port": "do-not-leak"}\')\n', "invalid value for port"), + ], +) +def test_credential_process_errors_do_not_expose_output( + tmp_path: Path, + body: str, + message: str, +): + command = _write_process(tmp_path, body) + config = load_config(_write_config(tmp_path, {"credential_process": command})) + + with pytest.raises(ValueError, match=message) as error: + build_executor(config) + + assert "do-not-leak" not in str(error.value) + assert error.value.__cause__ is None