From 10216173e1d10ce647566244593476c6cc879954 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 3 Nov 2026 12:30:42 +0000 Subject: [PATCH 1/2] Match the Python 0.15.1 behaviour in the Rust binary. Stdio MCP now answers newline-delimited JSON and still accepts Content-Length. CLI flags and directory scopes reach the engine, test detection covers the annotation and macro frameworks, and chisel migrate copies recorded results out of a legacy chisel.db. Benchmarks compare this build with Python 0.15.1. Co-authored-by: Aron Amos --- .grok/skills/chisel/SKILL.md | 6 +- AGENTS.md | 6 +- ARCHITECTURE.md | 4 +- GROK_CONNECTORS.md | 2 +- README.md | 45 ++- docs/CUSTOM_EXTRACTORS.md | 4 +- docs/ZERO_DEPS.md | 2 +- plan.md | 2 +- scripts/benchmark_parity.py | 270 ++++++++++++++++ scripts/benchmark_results.md | 66 ++++ skills/SKILL.md | 6 +- src/cli.rs | 283 ++++++++--------- src/cli_text.rs | 302 ++++++++++++++++++ src/engine/analyze.rs | 257 ++++++++++++--- src/engine/mod.rs | 30 +- src/engine/query.rs | 232 ++++++++++++-- src/extract/brace.rs | 13 +- src/extract/deps.rs | 104 ++++++- src/extract/framework.rs | 291 +++++++++++++++++ src/extract/mod.rs | 106 ++++++- src/git.rs | 16 + src/http.rs | 8 +- src/impact.rs | 85 +++++ src/lib.rs | 3 + src/mcp.rs | 73 +++-- src/next_steps.rs | 54 ++++ src/run_cmd.rs | 212 +++++++++++++ src/sqlite_import.rs | 468 ++++++++++++++++++++++++++++ tests/fixtures/legacy/chisel.sqlite | Bin 0 -> 12288 bytes tests/oracle_parity.rs | 2 +- tests/parity.rs | 339 ++++++++++++++++++++ 31 files changed, 3033 insertions(+), 258 deletions(-) create mode 100644 scripts/benchmark_parity.py create mode 100644 scripts/benchmark_results.md create mode 100644 src/cli_text.rs create mode 100644 src/extract/framework.rs create mode 100644 src/run_cmd.rs create mode 100644 src/sqlite_import.rs create mode 100644 tests/fixtures/legacy/chisel.sqlite create mode 100644 tests/parity.rs diff --git a/.grok/skills/chisel/SKILL.md b/.grok/skills/chisel/SKILL.md index 350c0b4..86a362c 100644 --- a/.grok/skills/chisel/SKILL.md +++ b/.grok/skills/chisel/SKILL.md @@ -7,7 +7,7 @@ description: Run Chisel test-impact analysis on any git project — which tests The binary is `chisel`, built from `src/` with `cargo build --release`. `chisel-mcp` is the same binary and starts stdio MCP. Oracle JSON is in `tests/fixtures/oracle`. Do not pip-install a package to get this binary. -Storage is the snapshot `.chisel/store`. Chisel does not open `chisel.db`. +Storage is the snapshot `.chisel/store`. `chisel migrate` copies test results from an old `chisel.db`. Other commands do not open it. ## Core loop @@ -20,7 +20,7 @@ chisel --json update chisel run -- pytest tests/ ``` -Prefer MCP when `chisel-mcp` is connected (26 tools). HTTP `chisel serve` is `GET /health`, `GET /tools`, and `POST /call` only. It is not an MCP URL and it has no `/mcp` path. +Prefer MCP when `chisel-mcp` is connected (26 tools). The server speaks newline-delimited JSON and also Content-Length. HTTP `chisel serve` is `GET /health`, `GET /tools`, and `POST /call` only. It is not an MCP URL and it has no `/mcp` path. Always pass `--json` on the CLI and branch on `status` before treating a payload as a list. `no_data`, `stale_db`, and `no_changes` exit 0. `error` and `git_error` exit 1. @@ -50,7 +50,7 @@ Always pass `--json` on the CLI and branch on `status` before treating a payload - A no-op `update` sets `edge_rebuild_skipped` true and does not run `git blame` or `git log -L`. - `chisel run` records pytest and Jest and forwards the test process exit code. One snapshot flush for the whole run. - `chisel watch` polls `git status` and runs `update` when the worktree changes. `--interval` is seconds. `--once` exits after one check. -- A custom parser is an executable named by `CHISEL_EXTRACTOR`, or by `.chisel/extensions.toml` when the process starts with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. A tool argument cannot turn it on. The path must stay inside the project. Its libraries stay outside this binary. +- A custom parser is an executable named by `CHISEL_EXTRACTOR`, or by `.chisel/extensions.toml` when the process starts with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. `CHISEL_DEP_EXTRACTOR` or `dep_extractor` in that file supplies test dependencies. A tool argument cannot turn the repo file on. The path must stay inside the project. Its libraries stay outside this binary. - `stats` returns counts. It does not embed every historical result. - List tools apply a default limit and set `truncated` when the cap cuts the list. - Suggestion `source` is one of `direct`, `co_change`, `import_graph`, `static_require`, `hybrid`, `fallback`, `working_tree`. diff --git a/AGENTS.md b/AGENTS.md index 785225f..7b9edb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ There is no Python package in this tree. Result checks use the JSON under `tests ## Storage -`.chisel/store` is the snapshot. Chisel does not open `chisel.db`. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` are honored when CLI flags are absent. `:memory:` and `file:` storage paths are rejected. +`.chisel/store` is the snapshot. `chisel migrate` is the only command that reads a legacy SQLite `chisel.db`, and it only copies `test_results` into the snapshot. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` are honoured when CLI flags are absent. `:memory:` and `file:` storage paths are rejected. ## How to call it @@ -25,7 +25,7 @@ CLI agents pass `--json` and branch on `status`. Exit code is not enough. `no_data` and `stale_db` exit 0. `error` and `git_error` exit 1. -MCP is `chisel serve-mcp` or the `chisel-mcp` binary, stdio Content-Length JSON-RPC. HTTP `chisel serve` answers `GET /health`, `GET /tools`, and `POST /call` only. There is no `/mcp` path. +MCP is `chisel serve-mcp` or the `chisel-mcp` binary. Stdio messages are newline-delimited JSON. Content-Length framing is still accepted and answered in the same framing. HTTP `chisel serve` answers `GET /health`, `GET /tools`, and `POST /call` only. There is no `/mcp` path. `suggest_tests` on a JS or TS file that calls `eval` or `new Function` adds a `warn_eval_used` next step. First call `analyze` when you see `no_data`. After edits, call `diff_impact`. Use `suggest_tests` with `file_path`. Use `triage` for a bounded risk pass. `impact` requires `files`. A missing required argument returns `status: "error"`, not an empty list. @@ -45,4 +45,4 @@ Scores are heuristics. Read `_meta.uniform_components` before trusting a composi Test filenames that produce edges include `test_*.py`, `*_test.py`, `*.test.ts`, `*.spec.ts`, `*_test.go`, `*_test.rs`, and any `.rs` file under the top-level `tests/` directory. Rust functions marked `#[test]`, `#[tokio::test]`, `#[rstest]`, or `#[test_case]` are test units even when the file is not named like a test. -`CHISEL_EXTRACTOR` or `extractor` in `.chisel/extensions.toml` names an executable. That program may use its own libraries. Chisel does not link them. The toml file is ignored unless the process was started with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. A tool argument cannot turn it on. The path must stay inside the project. `CHISEL_EXTRACTOR` overrides the file. A bad run falls back to the built-in scanner. A git ref that starts with `-` is rejected. +`CHISEL_EXTRACTOR` or `extractor` in `.chisel/extensions.toml` names an executable for code units. `CHISEL_DEP_EXTRACTOR` or `dep_extractor` in that file names an executable for test dependencies. It reads `{"path","language","content"}` and writes a JSON list of `{name,dep_type,module_path,confidence}`. That program may use its own libraries. Chisel does not link them. The toml file is ignored unless the process was started with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. A tool argument cannot turn it on. The path must stay inside the project. The environment variables override the file and do not need the flag. A bad run falls back to the built-in scanner. `CHISEL_BOOTSTRAP` is not imported as Python. If it is a path to an executable it is the dependency extractor. A git ref that starts with `-` is rejected. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4cb82b3..e976dda 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -28,7 +28,7 @@ The crate exposes two binaries. `chisel` is `src/main.rs`. `chisel-mcp` is `src/ | `src/json.rs` | Parser, serializer, Python-compatible rounding for scores. | | `src/hash.rs` | SHA-256 for file content. | | `src/http.rs` | The small JSON HTTP server. | -| `src/mcp.rs` | Content-Length framing and the tool bridge. | +| `src/mcp.rs` | Stdio MCP. Newline-delimited JSON, and Content-Length when the client sends that. | | `src/fswatch.rs` | OS change notification. `collect_files` is only compiled for the kqueue path. | | `src/lock.rs` | Cross-process shared and exclusive locks. | | `src/schemas.rs` | Tool list. Missing required arguments are `status: "error"`. | @@ -38,7 +38,7 @@ Shards are one snapshot each under `.chisel/shards//`. Writes use the lon ## Storage and git -`.chisel/store` is the snapshot. Chisel does not open `chisel.db`. `:memory:` and `file:` storage paths are rejected. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` apply when the CLI flags are absent. +`.chisel/store` is the snapshot. `chisel migrate` reads a legacy `chisel.db` once and copies `test_results`. Other commands do not open it. `:memory:` and `file:` storage paths are rejected. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` apply when the CLI flags are absent. Git is a subprocess. Locks are advisory records. They do not lock source files. diff --git a/GROK_CONNECTORS.md b/GROK_CONNECTORS.md index 176229d..b65cb4f 100644 --- a/GROK_CONNECTORS.md +++ b/GROK_CONNECTORS.md @@ -13,7 +13,7 @@ Point the client at the binary: - **command**: `chisel-mcp` (or `chisel serve-mcp`) - **env**: `CHISEL_PROJECT_DIR` set to the project root, and optionally `CHISEL_STORAGE_DIR` set to `/.chisel` -`chisel-mcp` speaks Content-Length JSON-RPC on stdin and stdout. The first tool call on an empty snapshot returns `status: "no_data"`. Call `analyze`, then `diff_impact` after edits. +`chisel-mcp` speaks newline-delimited JSON-RPC on stdin and stdout, and also accepts Content-Length framing. The first tool call on an empty snapshot returns `status: "no_data"`. Call `analyze`, then `diff_impact` after edits. ## HTTP diff --git a/README.md b/README.md index 411488c..061d57d 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,9 @@ cargo build --release The binary is `target/release/chisel`. `chisel serve-mcp` starts the stdio MCP server. A local build also writes `chisel-mcp`, which is the same program. -Storage is a snapshot file at `.chisel/store`. Chisel does not open `chisel.db`. Pass `--json` and branch on the `status` field. `no_data` means run `analyze`. `stale_db` means run `update` or pass `auto_update`. +Storage is a snapshot file at `.chisel/store`. Pass `--json` and branch on the `status` field. `no_data` means run `analyze`. `stale_db` means run `update` or pass `auto_update`. + +If you still have a SQLite `chisel.db` from the Python release, `chisel migrate path/to/chisel.db` copies the recorded test results into the snapshot. Other commands leave that file alone. The live store is not SQLite. ## Use with Claude Code (MCP) @@ -61,7 +63,7 @@ Add to your Claude Code MCP config (`~/.claude/settings.json` or project `.mcp.j } ``` -`chisel serve-mcp` speaks stdio MCP (Content-Length JSON-RPC). The release binary is `chisel`. A copy named `chisel-mcp` or `chisel-mcp.exe` starts the same server. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` are honored when the CLI flags are absent. +`chisel serve-mcp` speaks newline-delimited JSON-RPC on stdio, the same framing standard MCP clients send. A client that sends `Content-Length` headers still receives a `Content-Length` reply. `initialize` reports the crate version in `serverInfo.version`. The release binary is `chisel`. A copy named `chisel-mcp` or `chisel-mcp.exe` starts the same server. `CHISEL_PROJECT_DIR` and `CHISEL_STORAGE_DIR` are honoured when the CLI flags are absent. Run `analyze` first to build the project graph, then `diff_impact` after edits to see which tests to run. `analyze` with `force` on more than 300 code files returns `auto_queued` and a `job_id`. Poll `job_status`. `--working-tree` reads the worktree. Leave it off to answer from the snapshot. @@ -109,6 +111,12 @@ chisel run -- pytest tests/ # Incremental update (near-instant when nothing changed) chisel update + +# Copy recorded results out of a Python-era chisel.db +chisel migrate path/to/chisel.db + +# One command's flags +chisel churn --help ``` ## Try it on this repo @@ -134,8 +142,8 @@ The same binary has been run on flask, requests, cJSON, quixbugs, and the Probat | `analyze` | Full project scan — builds the code/test/git graph. Optional `shard` param for sharded monorepos | | `update` | Incremental re-analysis of changed files only. Optional `shard` param for sharded monorepos | | `diff_impact` | Detects your changes from `git diff` and returns impacted tests. `working_tree=true` reads untracked files. `auto_update=true` refreshes a stale snapshot inline | -| `suggest_tests` | Ranks tests by relevance for a given file. Requires `file_path`. `auto_update=true` refreshes a stale snapshot inline | -| `impact` | Which tests cover these files or functions? | +| `suggest_tests` | Ranks tests for one file (`file_path`) or every code file under `directory`. `fallback_to_all` ranks every known test when nothing else matches. A file that is not in the snapshot returns `stale_db` unless `working_tree` is set | +| `impact` | Which tests cover these files? Pass `functions` to limit the direct edges to those names | | `risk_map` | Risk scores for all files (churn + coupling + coverage gaps). Defaults: line-weighted coverage + proximity. `working_tree=true` includes untracked files. `exclude_new_file_boost=true` suppresses the new-file boost. `auto_update=true` refreshes a stale snapshot inline | | `test_gaps` | Code with zero test coverage, sorted by risk. `working_tree=true` includes uncommitted files. `auto_update=true` refreshes a stale snapshot inline | | `triage` | Top risks + gaps + stale tests in one call. Supports `exclude_new_file_boost` and `auto_update` | @@ -145,8 +153,8 @@ The same binary has been run on flask, requests, cJSON, quixbugs, and the Probat | `who_reviews` | Commit-activity-based — who maintains this code? | | `stale_tests` | Tests whose edges name code ids that are no longer in the snapshot | | `history` | Commit history for a file | -| `record_result` | Log test pass/fail outcomes for future prioritization | -| `run` | CLI-only: run tests and auto-record results (pytest, Jest) | +| `record_result` | Log a pass or fail. `duration_ms` is stored and feeds the duration-variance part of test instability | +| `run` | CLI-only: stream a test command and record pytest or Jest results. Jest is invoked with `--json` and a relative test id. `npx` is treated as Jest. `go test` gets `-json` and `cargo test` gets `--message-format=json` | | `stats` | Snapshot counts. It does not return the full result history | | `start_job` | Run analyze/update in background (avoids MCP timeouts). Optional `shard` param | | `job_status` | Poll a background job until complete | @@ -162,10 +170,10 @@ The same binary has been run on flask, requests, cJSON, quixbugs, and the Probat - **gitignore-aware** — ignored trees (vendored deps, build output, fixtures) are never scanned; untracked files still are (`CHISEL_INCLUDE_IGNORED=1` to override) - **Branch-aware** — `diff_impact` auto-detects feature branch vs main. Feature-branch co-change comes from `merge-base..HEAD`. `main` and `master` keep that list empty. - **Multi-agent safe** — cross-process locks so parallel agents don't corrupt the snapshot -- **MCP + CLI** — stdio MCP (`chisel serve-mcp`) and a JSON HTTP API (`GET /health`, `GET /tools`, `POST /call`), plus a CLI with 28 subcommands +- **MCP + CLI** — stdio MCP (`chisel serve-mcp`, newline JSON or Content-Length) and a JSON HTTP API (`GET /health`, `GET /tools`, `POST /call`). `chisel --help` prints that command. HTTP `suggest_tests` adds a warning when the file uses `eval` or `new Function` - **CI-friendly** — `error` and `git_error` exit 1. Other statuses, including `no_data` and `stale_db`, exit 0. `examples/github-actions/chisel.yml` builds this binary with Cargo. Do not pip-install Chisel. - **Monorepo sharding** — one snapshot per shard under `.chisel/shards//` (`CHISEL_SHARDS` or `.chisel/shards.toml`). Cross-shard test edges are overlaid from the parent snapshot. `flakiness_trend` appears when dated test results span two weeks. -- **Custom extractors** — set `CHISEL_EXTRACTOR` to an executable. It receives the path and language on stdin and returns a JSON list of units. A non-zero exit or bad JSON falls back to the built-in scanner and records a warning. Unset uses the built-in scanner. The binary does not load tree-sitter or plugins. +- **Custom extractors** — `CHISEL_EXTRACTOR` names an executable that returns code units. `CHISEL_DEP_EXTRACTOR`, or `dep_extractor` in `.chisel/extensions.toml`, names an executable that returns import and call dependencies. That replaces the old Python `register_dep_extractor` hook. `CHISEL_BOOTSTRAP` is not a Python import: if it points at an executable it is used as the dependency extractor, otherwise it is ignored with a warning. The binary does not load tree-sitter or plugins. ## Ecosystem @@ -175,6 +183,27 @@ Chisel sits in the agent loop: impact -> tests -> record results -> refresh anal Checked-in oracle JSON lives in `tests/fixtures/oracle`. There is no Python package to install. +## Benchmarks + +Timed on this machine: Linux 6.12.94+, 4 vCPUs, Intel Xeon, 15.6 GiB RAM, Rust 1.97.0, Python 3.12.3. Five runs per command. The figure is the median in milliseconds, with the minimum and maximum in parentheses. The small repo is a two-file Python sample. The large repo is the `django` package from django/django (about 1000 code files) plus 80 of its test modules, with history split into commits of at most 100 paths so a shallow-clone boundary commit does not pair every file with every other file. Peak RSS and the raw log are in [`scripts/benchmark_results.md`](scripts/benchmark_results.md). Reproduce with `scripts/benchmark_parity.py`. + +| Command | Python small | Rust small | Python large | Rust large | +|---|---:|---:|---:|---:| +| startup (`--help`) | 66.4 (64.9–74.1) | 0.3 (0.1–0.4) | 63.5 (62.6–72.3) | 0.4 (0.2–0.6) | +| cold analyse | 99.7 (96.7–118.2) | 7.5 (6.5–9.1) | 15088.1 (14562.3–15841.0) | 22393.5 (22178.9–22697.7) | +| warm analyse | 87.3 (86.6–96.7) | 7.8 (7.1–8.3) | 10358.8 (9935.4–10529.8) | 22332.3 (21923.2–22425.7) | +| warm update | 76.1 (71.1–83.1) | 2.7 (2.6–3.3) | 177.5 (163.3–215.0) | 190.9 (185.5–198.2) | +| impact | 67.3 (66.1–103.7) | 1.0 (0.8–1.1) | 125.3 (122.1–131.5) | 274.4 (255.0–281.6) | +| suggest-tests | 68.0 (65.5–98.9) | 1.0 (0.9–1.0) | 152.9 (146.3–155.2) | 263.7 (256.5–274.1) | +| risk-map | 69.2 (67.5–73.2) | 0.9 (0.8–0.9) | 283.7 (262.6–396.6) | 322.2 (313.5–416.8) | +| test-gaps | 69.6 (68.3–77.0) | 0.8 (0.7–1.0) | 210.6 (200.2–219.2) | 2815.4 (2720.7–3056.8) | +| triage | 66.9 (65.7–68.0) | 0.9 (0.9–1.0) | 293.4 (277.1–323.2) | 3091.5 (2908.0–3208.2) | +| churn | 65.5 (64.5–66.1) | 0.9 (0.8–1.0) | 80.4 (74.9–81.6) | 286.9 (268.0–294.8) | +| coupling | 67.5 (65.6–72.6) | 0.9 (0.8–1.0) | 85.8 (80.2–88.6) | 286.6 (260.3–334.0) | +| stats | 69.1 (65.1–71.6) | 0.8 (0.7–0.9) | 91.1 (78.9–97.2) | 264.3 (235.2–306.4) | + +On that large tree the Python snapshot had 829 test units and the Rust snapshot had 822. The small sample had 1 test unit on both sides. + ## License MIT diff --git a/docs/CUSTOM_EXTRACTORS.md b/docs/CUSTOM_EXTRACTORS.md index 2edf46c..324b303 100644 --- a/docs/CUSTOM_EXTRACTORS.md +++ b/docs/CUSTOM_EXTRACTORS.md @@ -14,6 +14,8 @@ A relative path is from the project root and must stay inside the project after Unset, with no file, uses the built-in scanner. -Storage is still `.chisel/store`. Do not open `chisel.db`. Do not use SQLite. +`CHISEL_DEP_EXTRACTOR`, or `dep_extractor` in the same toml file, is a second executable. It reads `{"path","language","content"}` on stdin and writes a JSON list of dependencies: `name`, `dep_type` (`import` or `call`), optional `module_path`, optional `confidence`. A bad run falls back to the built-in dependency scan. The same `--allow-repo-extractor` gate applies to the toml path. `CHISEL_DEP_EXTRACTOR` overrides the file. This is the replacement for Python `register_dep_extractor` and for `CHISEL_BOOTSTRAP`. Chisel does not import a Python module. If `CHISEL_BOOTSTRAP` is a path to an executable, that program is the dependency extractor. + +Storage is still `.chisel/store`. `chisel migrate` is the one-shot copy from an old `chisel.db`. Do not use SQLite as the live store. Status values: `no_data`, `no_changes`, `no_edges`, `git_error`, `stale_db`, `error`, `unsupported`, `auto_queued`. diff --git a/docs/ZERO_DEPS.md b/docs/ZERO_DEPS.md index 3bf7e04..b64aedb 100644 --- a/docs/ZERO_DEPS.md +++ b/docs/ZERO_DEPS.md @@ -2,6 +2,6 @@ `Cargo.toml` has empty `[dependencies]` and `[dev-dependencies]`. There is no `build.rs`. The binary does not pip-install anything and does not link SQLite. -Persistence is the snapshot `.chisel/store`. `chisel.db` is not opened. `:memory:` and `file:` storage paths are rejected. +Persistence is the snapshot `.chisel/store`. `chisel migrate` is the only reader of a legacy `chisel.db`, and it copies test results into the snapshot. `:memory:` and `file:` storage paths are rejected. Status values: `no_data`, `no_changes`, `no_edges`, `git_error`, `stale_db`, `error`, `unsupported`, `auto_queued`. diff --git a/plan.md b/plan.md index de9c68e..efe1947 100644 --- a/plan.md +++ b/plan.md @@ -26,7 +26,7 @@ Status values: `no_data`, `no_changes`, `no_edges`, `git_error`, `stale_db`, `er | `src/impact.rs` | Risk math. | | `src/json.rs` | JSON parser and serializer. | | `src/http.rs` | `GET /health`, `GET /tools`, `POST /call`. No `/mcp` path. | -| `src/mcp.rs` | Stdio Content-Length JSON-RPC. | +| `src/mcp.rs` | Stdio MCP: newline-delimited JSON, plus Content-Length. | | `src/fswatch.rs` | `inotify`, `kqueue`, or `ReadDirectoryChangesW`, then the caller checks git. | | `src/lock.rs` | `flock` on Unix, `LockFileEx` on Windows. | | `src/schemas.rs` | The 26 tool names and input schemas. | diff --git a/scripts/benchmark_parity.py b/scripts/benchmark_parity.py new file mode 100644 index 0000000..b42969e --- /dev/null +++ b/scripts/benchmark_parity.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Compare Chisel Python 0.15.1 with the Rust binary. + +Five runs per command. Prints median milliseconds and the min-max range, +plus peak RSS in kilobytes when /usr/bin/time can measure it. + +Usage: + python3 scripts/benchmark_parity.py \ + --rust target/release/chisel \ + --python /path/to/venv/bin/chisel \ + --small /tmp/chisel-bench-small \ + --large /tmp/django +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +RUNS = 5 + + +def machine_spec() -> dict[str, str]: + spec = { + "system": platform.platform(), + "python": platform.python_version(), + "cpu": platform.processor() or "unknown", + "machine": platform.machine(), + } + for key, cmd in ( + ("uname", ["uname", "-a"]), + ("cpu_model", ["bash", "-lc", "lscpu | awk -F: '/Model name/ {print $2; exit}'"]), + ("nproc", ["nproc"]), + ("mem_gb", ["bash", "-lc", "awk '/MemTotal/ {printf \"%.1f\", $2/1024/1024}' /proc/meminfo"]), + ("rustc", ["rustc", "--version"]), + ): + try: + spec[key] = subprocess.check_output(cmd, text=True, stderr=subprocess.DEVNULL).strip() + except (OSError, subprocess.CalledProcessError): + spec[key] = "unavailable" + return spec + + +def peak_rss_kb(pid: int) -> int | None: + try: + text = Path(f"/proc/{pid}/status").read_text() + except OSError: + return None + for line in text.splitlines(): + if line.startswith("VmHWM:"): + parts = line.split() + if len(parts) >= 2 and parts[1].isdigit(): + return int(parts[1]) + return None + + +def measure(cmd: list[str], cwd: Path, env: dict[str, str]) -> dict: + import threading + proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + rss = {"n": 0} + stop = threading.Event() + + def sample() -> None: + while not stop.is_set(): + value = peak_rss_kb(proc.pid) + if value: + rss["n"] = max(rss["n"], value) + stop.wait(0.01) + + thread = threading.Thread(target=sample, daemon=True) + thread.start() + start = time.perf_counter() + stdout, stderr = proc.communicate() + elapsed_ms = (time.perf_counter() - start) * 1000.0 + stop.set() + thread.join(timeout=0.2) + return { + "ms": elapsed_ms, + "rss_kb": rss["n"] or None, + "code": proc.returncode, + "stderr": (stderr or "")[-500:], + } + + +def summarise(samples: list[dict]) -> dict: + ok = [s for s in samples if s["code"] == 0] + if not ok: + return {"error": samples[-1]["stderr"] if samples else "no runs", "n": len(samples)} + values = sorted(s["ms"] for s in ok) + mid = values[len(values) // 2] + rss = [s["rss_kb"] for s in ok if s["rss_kb"] is not None] + return { + "median_ms": round(mid, 1), + "min_ms": round(values[0], 1), + "max_ms": round(values[-1], 1), + "n": len(ok), + "peak_rss_kb": max(rss) if rss else None, + } + + +def fmt(summary: dict) -> str: + if "error" in summary: + return "not measured" + return f"{summary['median_ms']:.1f} ({summary['min_ms']:.1f}–{summary['max_ms']:.1f})" + + +def prepare_small(path: Path) -> None: + if (path / ".git").exists(): + return + path.mkdir(parents=True, exist_ok=True) + (path / "src").mkdir(exist_ok=True) + (path / "tests").mkdir(exist_ok=True) + (path / "src" / "calc.py").write_text("def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - b\n") + (path / "tests" / "test_calc.py").write_text( + "from src.calc import add\n\ndef test_add():\n assert add(1, 1) == 2\n" + ) + env = os.environ.copy() + subprocess.check_call(["git", "init", "-b", "main"], cwd=path) + subprocess.check_call(["git", "config", "user.email", "bench@example.com"], cwd=path) + subprocess.check_call(["git", "config", "user.name", "Bench"], cwd=path) + subprocess.check_call(["git", "add", "-A"], cwd=path) + subprocess.check_call(["git", "commit", "-m", "init"], cwd=path, env={**env, "GIT_AUTHOR_DATE": "2024-01-01T00:00:00Z", "GIT_COMMITTER_DATE": "2024-01-01T00:00:00Z"}) + + +def run_suite(label: str, binary: str, project: Path, storage: Path, impact_file: str) -> dict[str, dict]: + env = os.environ.copy() + tail = ["--json", "--project-dir", str(project), "--storage-dir", str(storage)] + commands = { + "startup": [binary, "--help"], + "cold_analyze": [binary, "analyze", ".", *tail], + "warm_analyze": [binary, "analyze", ".", *tail], + "warm_update": [binary, "update", *tail], + "impact": [binary, "impact", impact_file, *tail], + "suggest_tests": [binary, "suggest-tests", impact_file, *tail], + "risk_map": [binary, "risk-map", *tail], + "test_gaps": [binary, "test-gaps", *tail], + "triage": [binary, "triage", *tail], + "churn": [binary, "churn", impact_file, *tail], + "coupling": [binary, "coupling", impact_file, *tail], + "stats": [binary, "stats", *tail], + } + out: dict[str, dict] = {} + for name, cmd in commands.items(): + samples = [] + for i in range(RUNS): + if name == "cold_analyze": + shutil.rmtree(storage, ignore_errors=True) + elif name != "startup" and not storage.exists() and name != "cold_analyze": + # Queries need a snapshot. Build one if a previous cold run was skipped. + subprocess.run([binary, "analyze", ".", *tail], cwd=project, env=env, capture_output=True, text=True) + samples.append(measure(cmd, project if name != "startup" else Path.cwd(), env)) + if samples[-1]["code"] != 0 and name != "startup": + break + out[name] = summarise(samples) + out[name]["label"] = label + print(f"{label} {name}: {fmt(out[name])}", flush=True) + return out + + +def snapshot_counts(storage: Path) -> str: + snap = storage / "store" + if snap.is_file(): + text = snap.read_text(errors="replace") + return f"rust snapshot bytes {len(text)}, test_units mentions {text.count('\"framework\"')}" + db = storage / "chisel.db" + if db.is_file(): + import sqlite3 + conn = sqlite3.connect(db) + try: + units = conn.execute("select count(*) from test_units").fetchone()[0] + edges = conn.execute("select count(*) from test_edges").fetchone()[0] + except sqlite3.Error as exc: + return f"python db unreadable: {exc}" + finally: + conn.close() + return f"python test_units {units}, test_edges {edges}" + return "no snapshot" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--rust", required=True) + parser.add_argument("--python", required=True) + parser.add_argument("--small", required=True) + parser.add_argument("--large", required=True) + parser.add_argument("--small-file", default="src/calc.py") + parser.add_argument("--large-file", default="django/http/request.py") + parser.add_argument("--out", default="scripts/benchmark_results.md") + args = parser.parse_args() + small = Path(args.small) + large = Path(args.large) + prepare_small(small) + if not large.is_dir(): + print(f"large repo missing: {large}", file=sys.stderr) + return 1 + spec = machine_spec() + rust_bin = str(Path(args.rust).resolve()) + py_bin = str(Path(args.python).resolve()) + results = { + "machine": spec, + "rust_bin": rust_bin, + "python_bin": py_bin, + "runs": RUNS, + "small": { + "rust": run_suite("rust-small", rust_bin, small, Path("/tmp/chisel-bench-small-rust"), args.small_file), + "python": run_suite("py-small", py_bin, small, Path("/tmp/chisel-bench-small-py"), args.small_file), + }, + "large": { + "rust": run_suite("rust-large", rust_bin, large, Path("/tmp/chisel-bench-large-rust"), args.large_file), + "python": run_suite("py-large", py_bin, large, Path("/tmp/chisel-bench-large-py"), args.large_file), + }, + } + lines = ["# Chisel benchmark results", "", "Generated by `scripts/benchmark_parity.py`. Every figure is from a real run.", "", "## Machine", ""] + for key, value in spec.items(): + lines.append(f"- {key}: {value}") + lines += ["", f"Rust binary: `{rust_bin}`", f"Python binary: `{py_bin}`", f"Runs per command: {RUNS}", ""] + lines.append("Times are median milliseconds, with the minimum and maximum in parentheses. Peak RSS is the highest VmHWM sampled from /proc during each run, in KiB.") + lines.append("") + header = "| Command | Python small | Rust small | Python large | Rust large |" + lines += [header, "|---|---:|---:|---:|---:|"] + order = ["startup", "cold_analyze", "warm_analyze", "warm_update", "impact", "suggest_tests", "risk_map", "test_gaps", "triage", "churn", "coupling", "stats"] + for name in order: + cells = [] + for side in ("small", "large"): + for impl in ("python", "rust"): + cells.append(fmt(results[side][impl].get(name, {"error": "missing"}))) + lines.append(f"| {name} | " + " | ".join(cells) + " |") + lines += ["", "## Peak RSS (KiB)", "", "| Command | Python small | Rust small | Python large | Rust large |", "|---|---:|---:|---:|---:|"] + for name in order: + cells = [] + for side in ("small", "large"): + for impl in ("python", "rust"): + summary = results[side][impl].get(name, {}) + rss = summary.get("peak_rss_kb") + cells.append(str(rss) if rss else "n/a") + lines.append(f"| {name} | " + " | ".join(cells) + " |") + lines += ["", "## Failures", ""] + any_fail = False + for side in ("small", "large"): + for impl in ("python", "rust"): + for name, summary in results[side][impl].items(): + if "error" in summary: + any_fail = True + lines.append(f"- {impl} {side} {name}: {summary['error'].strip()}") + if not any_fail: + lines.append("None.") + lines += ["", "## Snapshot counts after the timed runs", ""] + lines.append(f"- small rust: {snapshot_counts(Path('/tmp/chisel-bench-small-rust'))}") + lines.append(f"- small python: {snapshot_counts(Path('/tmp/chisel-bench-small-py'))}") + lines.append(f"- large rust: {snapshot_counts(Path('/tmp/chisel-bench-large-rust'))}") + lines.append(f"- large python: {snapshot_counts(Path('/tmp/chisel-bench-large-py'))}") + lines.append("") + lines.append("The small repo is a two-file Python sample created by this script. The large repo is the django package from django/django (about 1000 code files) plus 80 of its test modules. History is 38 commits of at most 100 paths each: a shallow clone's boundary commit lists every file, and pairing those files dominates co-change time, so the benchmark uses this bounded history of the same tree. Counts are taken from the snapshot left by the last timed command.") + text = "\n".join(lines) + "\n" + Path(args.out).write_text(text) + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark_results.md b/scripts/benchmark_results.md new file mode 100644 index 0000000..2e5df58 --- /dev/null +++ b/scripts/benchmark_results.md @@ -0,0 +1,66 @@ +# Chisel benchmark results + +Generated by `scripts/benchmark_parity.py`. Every figure is from a real run. + +## Machine + +- system: Linux-6.12.94+-x86_64-with-glibc2.39 +- python: 3.12.3 +- cpu: x86_64 +- machine: x86_64 +- uname: Linux cursor 6.12.94+ #1 SMP PREEMPT_DYNAMIC Mon Nov 2 21:34:19 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux +- cpu_model: Intel(R) Xeon(R) Processor +- nproc: 4 +- mem_gb: 15.6 +- rustc: rustc 1.97.0 (2d8144b78 2026-07-07) + +Rust binary: `/agent/repos/Chisel/target/release/chisel` +Python binary: `/tmp/chisel-py` +Runs per command: 5 + +Times are median milliseconds, with the minimum and maximum in parentheses. Peak RSS is the highest `VmHWM` sampled from `/proc//status` during each run, in KiB. + +| Command | Python small | Rust small | Python large | Rust large | +|---|---:|---:|---:|---:| +| startup | 66.4 (64.9–74.1) | 0.3 (0.1–0.4) | 63.5 (62.6–72.3) | 0.4 (0.2–0.6) | +| cold_analyze | 99.7 (96.7–118.2) | 7.5 (6.5–9.1) | 15088.1 (14562.3–15841.0) | 22393.5 (22178.9–22697.7) | +| warm_analyze | 87.3 (86.6–96.7) | 7.8 (7.1–8.3) | 10358.8 (9935.4–10529.8) | 22332.3 (21923.2–22425.7) | +| warm_update | 76.1 (71.1–83.1) | 2.7 (2.6–3.3) | 177.5 (163.3–215.0) | 190.9 (185.5–198.2) | +| impact | 67.3 (66.1–103.7) | 1.0 (0.8–1.1) | 125.3 (122.1–131.5) | 274.4 (255.0–281.6) | +| suggest_tests | 68.0 (65.5–98.9) | 1.0 (0.9–1.0) | 152.9 (146.3–155.2) | 263.7 (256.5–274.1) | +| risk_map | 69.2 (67.5–73.2) | 0.9 (0.8–0.9) | 283.7 (262.6–396.6) | 322.2 (313.5–416.8) | +| test_gaps | 69.6 (68.3–77.0) | 0.8 (0.7–1.0) | 210.6 (200.2–219.2) | 2815.4 (2720.7–3056.8) | +| triage | 66.9 (65.7–68.0) | 0.9 (0.9–1.0) | 293.4 (277.1–323.2) | 3091.5 (2908.0–3208.2) | +| churn | 65.5 (64.5–66.1) | 0.9 (0.8–1.0) | 80.4 (74.9–81.6) | 286.9 (268.0–294.8) | +| coupling | 67.5 (65.6–72.6) | 0.9 (0.8–1.0) | 85.8 (80.2–88.6) | 286.6 (260.3–334.0) | +| stats | 69.1 (65.1–71.6) | 0.8 (0.7–0.9) | 91.1 (78.9–97.2) | 264.3 (235.2–306.4) | + +## Peak RSS (KiB) + +| Command | Python small | Rust small | Python large | Rust large | +|---|---:|---:|---:|---:| +| startup | 23028 | 1084 | 22980 | 2228 | +| cold_analyze | 24508 | 2492 | 60740 | 84112 | +| warm_analyze | 24268 | 608 | 54708 | 130340 | +| warm_update | 23932 | 2048 | 27244 | 71280 | +| impact | 23700 | 2044 | 28124 | 107248 | +| suggest_tests | 23720 | 1624 | 34844 | 107328 | +| risk_map | 23772 | 476 | 52652 | 107284 | +| test_gaps | 23764 | 476 | 59708 | 107244 | +| triage | 23796 | 484 | 52564 | 108756 | +| churn | 23740 | 2252 | 23656 | 107324 | +| coupling | 23672 | 1792 | 23724 | 107308 | +| stats | 23692 | 1628 | 25800 | 107244 | + +## Failures + +None. + +## Snapshot counts after the timed runs + +- small rust: rust snapshot bytes 3923, test_units mentions 1 +- small python: python test_units 1, test_edges 2 +- large rust: rust snapshot bytes 8129361, test_units mentions 822 +- large python: python test_units 829, test_edges 10267 + +The small repo is a two-file Python sample created by this script. The large repo is the django package from django/django (about 1000 code files) plus 80 of its test modules. History is 38 commits of at most 100 paths each: a shallow clone's boundary commit lists every file, and pairing those files dominates co-change time, so the benchmark uses this bounded history of the same tree. Counts are taken from the snapshot left by the last timed command. diff --git a/skills/SKILL.md b/skills/SKILL.md index 350c0b4..86a362c 100644 --- a/skills/SKILL.md +++ b/skills/SKILL.md @@ -7,7 +7,7 @@ description: Run Chisel test-impact analysis on any git project — which tests The binary is `chisel`, built from `src/` with `cargo build --release`. `chisel-mcp` is the same binary and starts stdio MCP. Oracle JSON is in `tests/fixtures/oracle`. Do not pip-install a package to get this binary. -Storage is the snapshot `.chisel/store`. Chisel does not open `chisel.db`. +Storage is the snapshot `.chisel/store`. `chisel migrate` copies test results from an old `chisel.db`. Other commands do not open it. ## Core loop @@ -20,7 +20,7 @@ chisel --json update chisel run -- pytest tests/ ``` -Prefer MCP when `chisel-mcp` is connected (26 tools). HTTP `chisel serve` is `GET /health`, `GET /tools`, and `POST /call` only. It is not an MCP URL and it has no `/mcp` path. +Prefer MCP when `chisel-mcp` is connected (26 tools). The server speaks newline-delimited JSON and also Content-Length. HTTP `chisel serve` is `GET /health`, `GET /tools`, and `POST /call` only. It is not an MCP URL and it has no `/mcp` path. Always pass `--json` on the CLI and branch on `status` before treating a payload as a list. `no_data`, `stale_db`, and `no_changes` exit 0. `error` and `git_error` exit 1. @@ -50,7 +50,7 @@ Always pass `--json` on the CLI and branch on `status` before treating a payload - A no-op `update` sets `edge_rebuild_skipped` true and does not run `git blame` or `git log -L`. - `chisel run` records pytest and Jest and forwards the test process exit code. One snapshot flush for the whole run. - `chisel watch` polls `git status` and runs `update` when the worktree changes. `--interval` is seconds. `--once` exits after one check. -- A custom parser is an executable named by `CHISEL_EXTRACTOR`, or by `.chisel/extensions.toml` when the process starts with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. A tool argument cannot turn it on. The path must stay inside the project. Its libraries stay outside this binary. +- A custom parser is an executable named by `CHISEL_EXTRACTOR`, or by `.chisel/extensions.toml` when the process starts with `--allow-repo-extractor` or `CHISEL_ALLOW_REPO_EXTRACTOR=1`. `CHISEL_DEP_EXTRACTOR` or `dep_extractor` in that file supplies test dependencies. A tool argument cannot turn the repo file on. The path must stay inside the project. Its libraries stay outside this binary. - `stats` returns counts. It does not embed every historical result. - List tools apply a default limit and set `truncated` when the cap cuts the list. - Suggestion `source` is one of `direct`, `co_change`, `import_graph`, `static_require`, `hybrid`, `fallback`, `working_tree`. diff --git a/src/cli.rs b/src/cli.rs index fd26cd1..7d9cc5f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -5,7 +5,7 @@ use crate::json::{self, Value}; use std::env; use std::io::{self, Write}; use std::path::PathBuf; -use std::process::{Command, Stdio}; +use std::process::Command; pub struct Args { pub command: String, @@ -36,6 +36,15 @@ pub struct Args { pub once: bool, /// Seconds between worktree polls. Minimum 1. pub interval_secs: u64, + pub functions: Vec, + pub fallback: bool, + pub unit: Option, + pub min_count: Option, + pub no_exclude_tests: bool, + pub top_n: Option, + pub agent_id: Option, + pub duration_ms: Option, + pub help: bool, } pub fn parse(argv: &[String]) -> Result { @@ -64,6 +73,15 @@ pub fn parse(argv: &[String]) -> Result { let mut allow_repo_extractor = false; let mut once = false; let mut interval_secs = 2u64; + let mut functions = Vec::new(); + let mut fallback = false; + let mut unit = None; + let mut min_count = None; + let mut no_exclude_tests = false; + let mut top_n = None; + let mut agent_id = None; + let mut duration_ms = None; + let mut help = false; let mut i = 0; let mut after_dd = false; while i < argv.len() { @@ -157,6 +175,36 @@ pub fn parse(argv: &[String]) -> Result { if interval_secs == 0 { interval_secs = 1; } + } else if a == "--functions" { + i += 1; + while i < argv.len() && !argv[i].starts_with('-') { + functions.push(argv[i].clone()); + i += 1; + } + i = i.saturating_sub(1); + } else if a == "--fallback" { + fallback = true; + } else if a == "--unit" { + i += 1; + unit = Some(argv.get(i).ok_or("missing --unit")?.clone()); + } else if a == "--min-count" { + i += 1; + min_count = Some(argv.get(i).ok_or("missing --min-count")?.parse().map_err(|_| "bad min-count")?); + } else if a == "--no-exclude-tests" { + no_exclude_tests = true; + } else if a == "--proximity" { + proximity_adjustment = Some(true); + } else if a == "--top-n" { + i += 1; + top_n = Some(argv.get(i).ok_or("missing --top-n")?.parse().map_err(|_| "bad top-n")?); + } else if a == "--agent-id" { + i += 1; + agent_id = Some(argv.get(i).ok_or("missing --agent-id")?.clone()); + } else if a == "--duration" { + i += 1; + duration_ms = Some(argv.get(i).ok_or("missing --duration")?.parse().map_err(|_| "bad duration")?); + } else if a == "--help" || a == "-h" { + help = true; } else if a.starts_with("--") { return Err(format!("status=error unknown or unimplemented flag {a}")); } else if command.is_empty() { @@ -166,13 +214,14 @@ pub fn parse(argv: &[String]) -> Result { } i += 1; } - if command.is_empty() { + if command.is_empty() && !help { return Err("missing command".into()); } Ok(Args { command, project_dir, storage_dir, json, positionals, host, port, ttl, purpose, test_command, force, working_tree, auto_update, limit, shard, directory, git_ref, kind, passed, coverage_mode, proximity_adjustment, exclude_new_file_boost, allow_repo_extractor, once, interval_secs, + functions, fallback, unit, min_count, no_exclude_tests, top_n, agent_id, duration_ms, help, }) } @@ -208,7 +257,8 @@ pub fn run(args: &Args) -> Result { crate::mcp::serve(&args.project_dir, args.storage_dir.as_deref())?; return Ok(0); } - "run" => return run_tests(args), + "run" => return crate::run_cmd::run_tests(args), + "migrate" => return migrate(args), "watch" => return watch(args), _ => {} } @@ -249,9 +299,17 @@ fn call_args(args: &Args) -> Value { pairs.push(("passed".into(), Value::Bool(passed))); } } - "analyze" | "risk-map" | "test-gaps" => { + "analyze" | "risk-map" | "triage" => { pairs.push(("directory".into(), Value::String(first.clone()))); } + "test-gaps" => { + pairs.push(("file_path".into(), Value::String(first.clone()))); + } + "start-job" => { + if args.kind.is_none() && (first == "analyze" || first == "update") { + pairs.push(("kind".into(), Value::String(first.clone()))); + } + } "acquire-lock" | "release-lock" | "refresh-lock" => { pairs.push(("file_path".into(), Value::String(first.clone()))); if let Some(agent) = args.positionals.get(1) { @@ -297,6 +355,17 @@ fn call_args(args: &Args) -> Value { if let Some(kind) = &args.kind { pairs.push(("kind".into(), Value::String(kind.clone()))); + } else if args.command == "start-job" { + if let Some(kind) = args.positionals.iter().find(|p| *p == "analyze" || *p == "update") { + if !pairs.iter().any(|(k, _)| k == "kind") { + pairs.push(("kind".into(), Value::String(kind.clone()))); + } + } + if args.command == "start-job" { + if let Some(dir) = args.positionals.iter().find(|p| *p != "analyze" && *p != "update") { + pairs.push(("directory".into(), Value::String(dir.clone()))); + } + } } if let Some(mode) = &args.coverage_mode { pairs.push(("coverage_mode".into(), Value::String(mode.clone()))); @@ -307,6 +376,30 @@ fn call_args(args: &Args) -> Value { if args.exclude_new_file_boost { pairs.push(("exclude_new_file_boost".into(), Value::Bool(true))); } + if !args.functions.is_empty() { + pairs.push(("functions".into(), Value::Array(args.functions.iter().cloned().map(Value::String).collect()))); + } + if args.fallback { + pairs.push(("fallback_to_all".into(), Value::Bool(true))); + } + if let Some(unit) = &args.unit { + pairs.push(("unit_name".into(), Value::String(unit.clone()))); + } + if let Some(min_count) = args.min_count { + pairs.push(("min_count".into(), Value::Int(min_count))); + } + if args.no_exclude_tests { + pairs.push(("exclude_tests".into(), Value::Bool(false))); + } + if let Some(top_n) = args.top_n { + pairs.push(("top_n".into(), Value::Int(top_n))); + } + if let Some(agent) = &args.agent_id { + pairs.push(("agent_id".into(), Value::String(agent.clone()))); + } + if let Some(duration) = args.duration_ms { + pairs.push(("duration_ms".into(), Value::Int(duration))); + } if matches!(args.command.as_str(), "job-status" | "cancel-job") { if let Some(id) = args.positionals.first() { pairs.push(("job_id".into(), Value::String(id.clone()))); @@ -316,19 +409,7 @@ fn call_args(args: &Args) -> Value { } pub fn parse_jest_json(text: &str) -> Vec<(String, bool)> { - let Ok(doc) = json::parse(text) else { return Vec::new() }; - let Some(suites) = doc.get("testResults").and_then(|v| v.as_array()) else { return Vec::new() }; - let mut out = Vec::new(); - for suite in suites { - let name = suite.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let Some(assertions) = suite.get("assertionResults").and_then(|v| v.as_array()) else { continue }; - for assertion in assertions { - let title = assertion.get("title").and_then(|v| v.as_str()).unwrap_or(""); - let passed = assertion.get("status").and_then(|v| v.as_str()) == Some("passed"); - out.push((format!("{name}:{title}"), passed)); - } - } - out + crate::run_cmd::parse_jest_json(text, None) } fn emit(json_output: bool, command: &str, result: &Value) -> Result<(), String> { @@ -353,135 +434,9 @@ fn emit(json_output: bool, command: &str, result: &Value) -> Result<(), String> return Ok(()); } } - match command { - "analyze" | "update" | "stats" => { - let header = if command == "analyze" { "Analysis complete:" } else if command == "update" { "Update complete:" } else { "Stats:" }; - println!("{header}"); - if let Some(pairs) = result.as_object() { - for (k, v) in pairs { - println!(" {}: {}", title_key(k), fmt_scalar(v)); - } - } - } - "risk-map" => { - println!("Risk map:"); - if let Some(files) = result.get("files").and_then(|v| v.as_array()) { - for f in files { - let path = f.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); - let score = f.get("risk_score").and_then(|v| v.as_f64()).unwrap_or(0.0); - println!(" {path}: {score}"); - } - } - } - _ => { - println!("{}", json::to_string_pretty(result).map_err(|e| e.message)?); - } - } - Ok(()) + crate::cli_text::render(command, result) } -fn title_key(key: &str) -> String { - key.split('_').map(|w| { - let mut c = w.chars(); - match c.next() { - None => String::new(), - Some(f) => f.to_uppercase().collect::() + c.as_str(), - } - }).collect::>().join(" ") -} - -fn fmt_scalar(v: &Value) -> String { - match v { - Value::Int(i) => i.to_string(), - Value::Float(f) => f.to_string(), - Value::Bool(b) => b.to_string(), - Value::String(s) => s.clone(), - Value::Null => "null".into(), - other => json::to_string(other).unwrap_or_else(|_| "…".into()), - } -} - -fn run_tests(args: &Args) -> Result { - if args.test_command.is_empty() { - return Err("chisel run requires a command after --".into()); - } - let mut cmd_args = args.test_command.clone(); - let program = cmd_args.remove(0); - let framework = if program == "pytest" || program.ends_with("/pytest") { - "pytest" - } else if program == "jest" || program.ends_with("/jest") { - "jest" - } else { - "" - }; - if framework == "pytest" && !cmd_args.iter().any(|a| a == "-v" || a == "--verbose") { - cmd_args.push("-v".into()); - } - let mut child = Command::new(&program) - .args(&cmd_args) - .current_dir(&args.project_dir) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| e.to_string())?; - let mut stdout = child.stdout.take(); - let mut stderr = child.stderr.take(); - let mut out_buf = Vec::new(); - let mut err_buf = Vec::new(); - if let Some(ref mut o) = stdout { - use std::io::Read; - let _ = o.read_to_end(&mut out_buf); - } - if let Some(ref mut e) = stderr { - use std::io::Read; - let _ = e.read_to_end(&mut err_buf); - } - let _ = io::stdout().write_all(&out_buf); - let _ = io::stderr().write_all(&err_buf); - let status = child.wait().map_err(|e| e.to_string())?; - let code = status.code().unwrap_or(1); - if framework == "pytest" || framework == "jest" { - let text = String::from_utf8_lossy(&out_buf); - let rows = if framework == "jest" { - parse_jest_json(&text) - } else { - text.lines().filter_map(parse_pytest_line).collect() - }; - let mut engine = Engine::open(&args.project_dir, args.storage_dir.as_deref())?; - let recorded = rows.len(); - if recorded > 0 { - engine.record_results_batch(&rows)?; - println!("\n[chisel] Recorded {recorded} test result(s)"); - } else { - println!("\n[chisel] No test results parsed from output"); - } - } - Ok(code) -} - -fn parse_pytest_line(line: &str) -> Option<(String, bool)> { - let line = line.trim_end(); - let mut parts = line.rsplitn(2, ' '); - let status = parts.next()?.trim(); - let status = status.split_whitespace().next()?; - if !matches!(status, "PASSED" | "FAILED" | "ERROR" | "SKIPPED" | "XFAIL" | "XPASS") { - // progress suffix: "PASSED [ 50%]" - let tokens: Vec<&str> = line.split_whitespace().collect(); - let idx = tokens.iter().position(|t| matches!(*t, "PASSED" | "FAILED" | "ERROR" | "SKIPPED" | "XFAIL" | "XPASS"))?; - if idx == 0 { - return None; - } - let id = tokens[..idx].join(" "); - let passed = matches!(tokens[idx], "PASSED" | "XFAIL"); - return Some((id, passed)); - } - let id = parts.next()?.trim().to_string(); - if id.is_empty() { - return None; - } - let passed = matches!(status, "PASSED" | "XFAIL"); - Some((id, passed)) -} fn worktree_fingerprint(dir: &PathBuf) -> Result { let status = Command::new("git") @@ -590,19 +545,55 @@ pub fn entry() -> Result { if argv_is_mcp_binary(&argv0) { argv.insert(0, "serve-mcp".into()); } - if argv.is_empty() || argv.iter().any(|a| a == "-h" || a == "--help") && argv.len() == 1 { + if argv.is_empty() { help(); return Ok(0); } let args = parse(&argv)?; + if args.help { + if args.command.is_empty() { + help(); + } else { + crate::cli_text::command_help(&args.command); + } + return Ok(0); + } run(&args) } +fn migrate(args: &Args) -> Result { + let db = if let Some(path) = args.positionals.first() { + PathBuf::from(path) + } else { + let candidates = [ + args.project_dir.join("chisel.db"), + args.project_dir.join(".chisel").join("chisel.db"), + args.storage_dir.clone().unwrap_or_else(|| args.project_dir.join(".chisel")).join("chisel.db"), + ]; + match candidates.into_iter().find(|p| p.is_file()) { + Some(path) => path, + None => { + let body = Value::object(vec![ + ("status".into(), Value::String("error".into())), + ("message".into(), Value::String("No chisel.db found. Pass the path: chisel migrate path/to/chisel.db".into())), + ]); + emit(args.json, "migrate", &body)?; + return Ok(1); + } + } + }; + let mut engine = Engine::open(&args.project_dir, args.storage_dir.as_deref())?; + let result = engine.migrate_legacy(&db)?; + emit(args.json, "migrate", &result)?; + Ok(0) +} + pub fn help() { println!("Chisel — test impact analysis and code intelligence"); println!("usage: chisel [--json] [--version] [--project-dir DIR] [--storage-dir DIR] ..."); println!("commands: analyze, impact, suggest-tests, churn, ownership, coupling, risk-map,"); println!(" stale-tests, history, who-reviews, diff-impact, update, test-gaps, record-result,"); println!(" run, watch, stats, start-job, job-status, cancel-job, triage, serve, serve-mcp,"); - println!(" acquire-lock, release-lock, refresh-lock, check-lock, check-locks, list-locks"); + println!(" migrate, acquire-lock, release-lock, refresh-lock, check-lock, check-locks, list-locks"); + println!("Run `chisel --help` for one command."); } diff --git a/src/cli_text.rs b/src/cli_text.rs new file mode 100644 index 0000000..61323f5 --- /dev/null +++ b/src/cli_text.rs @@ -0,0 +1,302 @@ +//! Human-readable command output and per-command help. + +use crate::json::{self, Value}; + +pub fn render(command: &str, result: &Value) -> Result<(), String> { + match command { + "impact" | "diff-impact" => print_list(result, "No impacted tests found.", if command == "impact" { "Impacted tests:" } else { "Impacted tests from diff:" }, |item| { + format!("{} ({})", item.get("test_id").and_then(|v| v.as_str()).unwrap_or(""), item.get("reason").and_then(|v| v.as_str()).unwrap_or("")) + }), + "suggest-tests" => render_suggest(result), + "churn" => render_churn(result), + "ownership" => print_list(result, "No ownership data.", "Ownership:", |item| { + format!("{}: {}", item.get("author").and_then(|v| v.as_str()).unwrap_or(""), scalar(item.get("percentage"))) + }), + "coupling" => render_coupling(result), + "risk-map" => render_risk(result), + "stale-tests" => print_list(result, "No stale tests found.", "Stale tests:", |item| { + format!("{} ({})", item.get("test_id").and_then(|v| v.as_str()).unwrap_or(""), item.get("edge_type").and_then(|v| v.as_str()).unwrap_or("")) + }), + "history" => print_list(result, "No history found.", "History:", |item| { + let hash = item.get("hash").and_then(|v| v.as_str()).unwrap_or(""); + let short = hash.get(..8).unwrap_or(hash); + format!("{short} {} {} {}", scalar(item.get("date")), scalar(item.get("author")), scalar(item.get("message"))) + }), + "who-reviews" => print_list(result, "No reviewer suggestions.", "Suggested reviewers:", |item| { + format!("{}: {}% ({} commits, {}d ago)", scalar(item.get("author")), scalar(item.get("percentage")), scalar(item.get("recent_commits")), scalar(item.get("days_since_last_commit"))) + }), + "test-gaps" => print_list(result, "No untested code units found.", "Untested code units:", |item| { + format!("{}:{} ({}, lines {}-{}, churn: {})", scalar(item.get("file_path")), scalar(item.get("name")), scalar(item.get("unit_type")), scalar(item.get("line_start")), scalar(item.get("line_end")), scalar(item.get("churn_score"))) + }), + "record-result" => { + let id = result.get("test_id").and_then(|v| v.as_str()).unwrap_or(""); + let passed = result.get("passed").and_then(|v| v.as_bool()).unwrap_or(false); + println!("Recorded: {id} — {}", if passed { "PASSED" } else { "FAILED" }); + } + "triage" => render_triage(result), + "analyze" | "update" | "stats" | "start-job" | "job-status" | "cancel-job" | "migrate" => { + let header = match command { + "analyze" => "Analysis complete:", + "update" => "Incremental update complete:", + "stats" => "Chisel database stats:", + "start-job" => "Background job:", + "job-status" => "Job status:", + "cancel-job" => "Cancel job:", + _ => "Migrate:", + }; + println!("{header}"); + if let Some(pairs) = result.as_object() { + for (k, v) in pairs { + println!(" {}: {}", title_key(k), scalar(Some(v))); + } + } + } + "acquire-lock" | "release-lock" | "refresh-lock" | "check-lock" | "check-locks" | "list-locks" => render_lock(command, result), + _ => println!("{}", json::to_string_pretty(result).map_err(|e| e.message)?), + } + Ok(()) +} + +fn render_suggest(result: &Value) { + if let Some(pairs) = result.as_object() { + if pairs.is_empty() { + println!("No test suggestions."); + return; + } + for (file, items) in pairs { + println!("\n{file}:"); + if let Some(rows) = items.as_array() { + for item in rows { + println!(" {} (score: {})", scalar(item.get("name")), scalar(item.get("relevance"))); + } + } + } + return; + } + print_list(result, "No test suggestions.", "Suggested tests:", |item| { + format!("{} (score: {})", scalar(item.get("name")), scalar(item.get("relevance"))) + }); +} + +fn render_churn(result: &Value) { + let Some(rows) = result.as_array() else { + println!("{}", json::to_string_pretty(result).unwrap_or_default()); + return; + }; + if rows.is_empty() { + println!("No churn data available."); + return; + } + let file = rows.first().and_then(|r| r.get("file_path")).and_then(|v| v.as_str()).unwrap_or(""); + println!("Churn stats for {file}:"); + for item in rows { + if let Some(pairs) = item.as_object() { + for (k, v) in pairs { + println!(" {k}: {}", scalar(Some(v))); + } + println!(); + } + } +} + +fn render_coupling(result: &Value) { + let cc = result.get("co_change_partners").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); + let imp = result.get("import_partners").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); + if cc.is_empty() && imp.is_empty() { + println!("No coupling data."); + return; + } + if !cc.is_empty() { + println!("Co-change partners:"); + for p in &cc { + println!(" {} ({} co-commits)", scalar(p.get("file_b").or_else(|| p.get("file_a"))), scalar(p.get("co_commit_count"))); + } + } + if !imp.is_empty() { + println!("Import partners:"); + for p in &imp { + println!(" {}", scalar(p.get("file"))); + } + } + if let Some(note) = result.get("note").and_then(|v| v.as_str()) { + println!("{note}"); + } +} + +fn render_risk(result: &Value) { + let files = result.get("files").and_then(|v| v.as_array()); + let Some(files) = files else { + println!("No risk data."); + return; + }; + if files.is_empty() { + println!("No risk data."); + return; + } + println!("Risk map:"); + for item in files { + let locked = item.get("locked_by").and_then(|v| v.as_str()).map(|a| format!(" locked by {a}")).unwrap_or_default(); + println!(" {}: {}{locked}", scalar(item.get("file_path")), scalar(item.get("risk_score"))); + } + if let Some(uniform) = result.get("_meta").and_then(|m| m.get("uniform_components")).and_then(|v| v.as_object()) { + if !uniform.is_empty() { + println!("\nDiagnostics (uniform components — not differentiating):"); + for (comp, info) in uniform { + println!(" {comp}: {}", info.get("reason").and_then(|v| v.as_str()).unwrap_or("")); + } + } + } + if let Some(cycles) = result.get("_meta").and_then(|m| m.get("cycles")).and_then(|v| v.as_array()) { + if !cycles.is_empty() { + println!("\nImport cycles:"); + for cycle in cycles { + let files = cycle.get("files").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|v| v.as_str()).collect::>().join(", ")).unwrap_or_default(); + println!(" {files}"); + } + } + } +} + +fn render_triage(result: &Value) { + let Some(summary) = result.get("summary") else { + println!("{}", json::to_string_pretty(result).unwrap_or_default()); + return; + }; + println!("Triage ({} files):", scalar(summary.get("files_triaged"))); + println!("\nTop risk files:"); + if let Some(rows) = result.get("top_risk_files").and_then(|v| v.as_array()) { + for row in rows { + println!(" {}: {}", scalar(row.get("file_path")), scalar(row.get("risk_score"))); + } + } + if let Some(gaps) = result.get("test_gaps").and_then(|v| v.as_array()) { + if gaps.is_empty() { + println!("\nNo test gaps in triaged files."); + } else { + println!("\nTest gaps ({}):", scalar(summary.get("total_test_gaps"))); + for gap in gaps { + println!(" {}:{} ({})", scalar(gap.get("file_path")), scalar(gap.get("name")), scalar(gap.get("unit_type"))); + } + } + } + if let Some(stale) = result.get("stale_tests").and_then(|v| v.as_array()) { + if stale.is_empty() { + println!("\nNo stale tests found."); + } else { + println!("\nStale tests ({}):", scalar(summary.get("total_stale_tests"))); + for row in stale { + println!(" {} ({})", scalar(row.get("test_id")), scalar(row.get("edge_type"))); + } + } + } +} + +fn render_lock(command: &str, result: &Value) { + match command { + "list-locks" => { + let locks = result.get("locks").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); + if locks.is_empty() { + println!("No active locks"); + } else { + println!("{} active lock(s):", scalar(result.get("total"))); + for lock in locks { + let stale = if lock.get("ttl_remaining").and_then(|v| v.as_f64()).unwrap_or(0.0) < 60.0 { " (STALE)" } else { "" }; + println!(" {}: {} ({}s remaining){stale}", scalar(lock.get("file_path")), scalar(lock.get("agent_id")), scalar(lock.get("ttl_remaining"))); + } + } + } + "check-locks" => { + let conflicts = result.get("conflicts").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); + if conflicts.is_empty() { + println!("No locks on {} checked file(s)", scalar(result.get("checked"))); + } else { + println!("{} lock(s) found:", conflicts.len()); + for row in conflicts { + println!(" {}: {}", scalar(row.get("file_path")), scalar(row.get("holder"))); + } + } + } + "check-lock" => { + if result.get("locked").and_then(|v| v.as_bool()) == Some(true) { + println!("Locked"); + println!("Holder: {}", scalar(result.get("holder"))); + println!("TTL rem: {}s", scalar(result.get("ttl_remaining"))); + println!("Stale: {}", scalar(result.get("stale"))); + } else { + println!("Unlocked"); + } + } + _ => { + println!("{}", json::to_string_pretty(result).unwrap_or_default()); + } + } +} + +fn print_list(result: &Value, empty: &str, header: &str, line: impl Fn(&Value) -> String) { + let Some(rows) = result.as_array() else { + println!("{}", json::to_string_pretty(result).unwrap_or_default()); + return; + }; + if rows.is_empty() { + println!("{empty}"); + return; + } + println!("{header}"); + for item in rows { + println!(" {}", line(item)); + } +} + +fn scalar(v: Option<&Value>) -> String { + match v { + Some(Value::Int(i)) => i.to_string(), + Some(Value::Float(f)) => f.to_string(), + Some(Value::Bool(b)) => b.to_string(), + Some(Value::String(s)) => s.clone(), + Some(Value::Null) | None => "null".into(), + Some(other) => json::to_string(other).unwrap_or_else(|_| "…".into()), + } +} + +pub fn title_key(key: &str) -> String { + key.split('_').map(|w| { + let mut c = w.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } + }).collect::>().join(" ") +} + +pub fn command_help(command: &str) { + let text = match command { + "analyze" => "analyze [DIR] [--force] [--shard NAME]\n Scan the project and write .chisel/store.", + "update" => "update [--shard NAME]\n Recompute changed files. A no-op reports code units found as 0.", + "impact" => "impact FILE... [--functions NAME...]\n Tests that reach the named files, optionally limited to those functions.", + "suggest-tests" => "suggest-tests FILE | --directory DIR [--fallback] [--working-tree] [--auto-update]\n Rank tests for one file, or return a map for every code file under a directory.", + "churn" => "churn FILE [--unit NAME]\n File-level churn, or one unit row with insertions and last changed.", + "ownership" => "ownership FILE\n Blame share for a file.", + "coupling" => "coupling FILE [--min-count N] [--working-tree]\n Co-change partners. --working-tree scans an untracked file on disk.", + "risk-map" => "risk-map [DIR] [--no-exclude-tests] [--proximity|--no-proximity] [--coverage-mode unit|line]\n Risk scores. DIR limits the files. Cycles and locks are included when present.", + "test-gaps" => "test-gaps [FILE] [--directory DIR] [--no-exclude-tests]\n Untested units. A file argument or --directory limits the scope.", + "triage" => "triage [DIR] [--top-n N] [--no-exclude-tests]\n Top risk files plus their gaps and stale tests.", + "stale-tests" => "stale-tests\n Tests whose edges point at code that is gone. Returns a list when edges exist.", + "record-result" => "record-result TEST_ID (--passed|--failed) [--duration MS]\n Store a result. Duration feeds the instability variance.", + "run" => "run -- COMMAND...\n Stream pytest, jest/npx, go test, or cargo test and record pytest and Jest results.", + "stats" => "stats\n Counts, including co_change_query_min as a number and the shadow graph.", + "start-job" => "start-job analyze|update [DIR] [--force] [--shard NAME]\n Queue analyze or update. The kind may be the first argument.", + "job-status" => "job-status JOB_ID", + "cancel-job" => "cancel-job JOB_ID", + "list-locks" => "list-locks [--agent-id ID]\n Active advisory locks, optionally for one agent.", + "serve" => "serve [--host HOST] [--port PORT]\n HTTP JSON API. Not an MCP URL.", + "serve-mcp" => "serve-mcp\n Stdio MCP. Accepts newline-delimited JSON and Content-Length framing.", + "migrate" => "migrate [chisel.db]\n Copy test results from a legacy SQLite chisel.db into .chisel/store.", + "watch" => "watch [--once] [--interval SECS]\n Run update when the worktree changes.", + other => { + println!("Unknown command '{other}'."); + println!("Run `chisel --help` for the command list."); + return; + } + }; + println!("chisel {text}"); +} diff --git a/src/engine/analyze.rs b/src/engine/analyze.rs index f970f69..6df5022 100644 --- a/src/engine/analyze.rs +++ b/src/engine/analyze.rs @@ -123,10 +123,8 @@ impl Engine { if func_commits.is_empty() { continue; } - let func_dates: Vec = func_commits.iter().map(|c| c.date.clone()).collect(); - let func_score = impact::churn_score(&func_dates, now); - let func_count = func_dates.iter().filter(|d| impact::parse_iso_unix(d).is_some()).count() as i64; - churn_rows.push(churn_row(&task.0, &task.1, ChurnCounts { commits: func_count, authors: 1, insertions: 0, deletions: 0 }, "", func_score)); + let (func_count, author_n, ins, del, last, func_score) = unit_churn_from(&func_commits, &task.0); + churn_rows.push(churn_row(&task.0, &task.1, ChurnCounts { commits: func_count, authors: author_n, insertions: ins, deletions: del }, &last, func_score)); } } self.store.doc.insert("churn", Value::Array(churn_rows)); @@ -234,11 +232,17 @@ impl Engine { let new_commits: Vec = git.parse_log_since(since.as_deref()).unwrap_or_default().into_iter().filter(|c| !known.contains(&c.hash)).collect(); let commit_touch: BTreeSet = new_commits.iter().flat_map(|c| c.files.iter().map(|f| f.path.clone())).collect(); if changed.is_empty() && new_commits.is_empty() && removed.is_empty() { + let cleaned = self.cleanup_orphaned_results(); + if cleaned > 0 { + self.store.save()?; + } return Ok(Value::object(vec![ ("files_updated".into(), Value::Int(0)), - ("code_units_found".into(), Value::Int(self.store.array("code_units").len() as i64)), + ("code_units_found".into(), Value::Int(0)), ("new_commits".into(), Value::Int(0)), ("edge_rebuild_skipped".into(), Value::Bool(true)), + ("orphaned_results_cleaned".into(), Value::Int(cleaned)), + ("active_file_locks".into(), Value::Int(self.active_locks().len() as i64)), ("hint".into(), Value::String("For large repos, use start_job with kind='update' to avoid MCP timeouts.".into())), ])); } @@ -372,10 +376,8 @@ impl Engine { if func_commits.is_empty() { continue; } - let func_dates: Vec = func_commits.iter().map(|c| c.date.clone()).collect(); - let func_score = impact::churn_score(&func_dates, now); - let func_count = func_dates.iter().filter(|d| impact::parse_iso_unix(d).is_some()).count() as i64; - churn_rows.push(churn_row(&task.0, &task.1, ChurnCounts { commits: func_count, authors: 1, insertions: 0, deletions: 0 }, "", func_score)); + let (func_count, author_n, ins, del, last, func_score) = unit_churn_from(&func_commits, &task.0); + churn_rows.push(churn_row(&task.0, &task.1, ChurnCounts { commits: func_count, authors: author_n, insertions: ins, deletions: del }, &last, func_score)); } } self.store.doc.insert("churn", Value::Array(churn_rows)); @@ -489,11 +491,15 @@ impl Engine { self.store.doc.insert("import_edges", Value::Array(imports)); self.drop_removed_paths(removed); self.store.save()?; + let cleaned = self.cleanup_orphaned_results(); + let found = changed.iter().map(|rel| units_by_file.get(rel).map(|u| u.len() as i64).unwrap_or(0)).sum::(); let mut summary = vec![ ("files_updated".into(), Value::Int((changed.len() + removed.len()) as i64)), - ("code_units_found".into(), Value::Int(self.store.array("code_units").len() as i64)), + ("code_units_found".into(), Value::Int(found)), ("new_commits".into(), Value::Int(new_commits.len() as i64)), ("edge_rebuild_skipped".into(), Value::Bool(edge_rebuild_skipped)), + ("orphaned_results_cleaned".into(), Value::Int(cleaned)), + ("active_file_locks".into(), Value::Int(self.active_locks().len() as i64)), ("hint".into(), Value::String("For large repos, use start_job with kind='update' to avoid MCP timeouts.".into())), ]; if skip_unit { @@ -563,29 +569,30 @@ impl Engine { let mut test_units = Vec::new(); for rel in files { let units = units_by_file.get(rel).cloned().unwrap_or_default(); - let as_file = looks_like_test(rel); - let framework = if rel.ends_with(".py") { - "pytest" - } else if rel.ends_with(".go") { - "go" - } else { - "test" - }; - let selected: Vec = if as_file { - units.into_iter().filter(|u| { - extract::is_test_name(&u.name, framework) || is_test_unit_type(&u.unit_type) - }).collect() - } else { - units.into_iter().filter(|u| is_test_unit_type(&u.unit_type)).collect() - }; - if !as_file && selected.is_empty() { - continue; - } let abs = self.project_dir.join(rel); let content = fs::read_to_string(&abs).unwrap_or_default(); self.source_reads += 1; + let framework = extract::framework_for(rel, &content).unwrap_or(if looks_like_test(rel) { "test" } else { "" }); + let annotated = extract::annotated_names(&content, framework); + let as_file = looks_like_test(rel) || framework == "gtest"; + let selected: Vec = units.into_iter().filter(|u| { + if is_test_unit_type(&u.unit_type) { + return as_file || framework == "rust" || framework == "gtest" || framework == "jest" || framework == "playwright"; + } + if !as_file { + return false; + } + let bare = u.name.rsplit('.').next().unwrap_or(&u.name); + annotated.contains(bare) || annotated.contains(&u.name) || extract::is_test_name(&u.name, framework) + }).collect(); + if !as_file && selected.is_empty() { + continue; + } let hash = sha256_hex(content.as_bytes()); if selected.is_empty() { + if matches!(framework, "junit" | "csharp_test" | "xctest" | "gtest") { + continue; + } let stem = rel.rsplit(['/', '\\']).next().unwrap_or(rel); test_units.push(Value::object(vec![ ("id".into(), Value::String(format!("{rel}:{stem}"))), @@ -616,19 +623,18 @@ impl Engine { let mut static_seen = BTreeSet::new(); let mut seen = BTreeSet::new(); let mut content_cache: BTreeMap = BTreeMap::new(); - let mut bare_indexes: BTreeMap>> = BTreeMap::new(); + let shared_index = bare_name_index(&code_units); + if !test_units.is_empty() { + self.bare_index_builds += 1; + } for tu in &test_units { let file = tu.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); let tid = tu.get("id").and_then(|v| v.as_str()).unwrap_or(""); - if !bare_indexes.contains_key(file) { - self.bare_index_builds += 1; - bare_indexes.insert(file.to_string(), bare_name_index(&code_units)); - } - let index = bare_indexes.get(file).expect("bare-name index"); + let index = &shared_index; let content = content_cache.entry(file.to_string()).or_insert_with(|| { fs::read_to_string(self.project_dir.join(file)).unwrap_or_default() }); - let deps = extract::extract_deps(file, content); + let deps = extract::extract_deps_in(file, content, Some(&self.project_dir), self.allow_repo_extractor); let resolved = resolved_import_files(&deps, &code_units); for path in &resolved { let key = format!("{file}|{path}"); @@ -643,6 +649,9 @@ impl Engine { let matched = match_dep(file, &dep, &code_units, &resolved, index); for cid in matched { let code_file = code_units.iter().find(|u| u.get("id").and_then(|v| v.as_str()) == Some(cid.as_str())).and_then(|u| u.get("file_path").and_then(|v| v.as_str())).unwrap_or(""); + if code_file.is_empty() || code_file == file { + continue; + } let key = format!("{tid}|{cid}|{}", dep.dep_type); if !seen.insert(key) { continue; @@ -657,11 +666,19 @@ impl Engine { } } } + backfill_heuristic(&test_units, &mut edges, &code_units); (test_units, edges, static_rows) } pub(super) fn build_import_edges(&self, files: &[String]) -> Vec { let code_units = self.store.array("code_units"); + let mut by_name: BTreeMap<&str, Vec> = BTreeMap::new(); + for (i, cu) in code_units.iter().enumerate() { + let name = cu.get("name").and_then(|v| v.as_str()).unwrap_or(""); + if !name.is_empty() { + by_name.entry(name).or_default().push(i); + } + } let mut edges = Vec::new(); let mut seen = BTreeSet::new(); for rel in files { @@ -669,15 +686,16 @@ impl Engine { continue; } let content = fs::read_to_string(self.project_dir.join(rel)).unwrap_or_default(); - for dep in extract::extract_deps(rel, &content) { + for dep in extract::extract_deps_in(rel, &content, Some(&self.project_dir), self.allow_repo_extractor) { if dep.dep_type != "import" && dep.dep_type != "tainted_import" { continue; } let Some(module) = dep.module_path.as_deref() else { continue }; - for cu in &code_units { + let Some(hits) = by_name.get(dep.name.as_str()) else { continue }; + for &i in hits { + let cu = &code_units[i]; let path = cu.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); - let name = cu.get("name").and_then(|v| v.as_str()).unwrap_or(""); - if name == dep.name && extract::matches_import_path(path, module) && path != rel { + if extract::matches_import_path(path, module) && path != rel { let key = format!("{rel}|{path}"); if seen.insert(key) { edges.push(Value::object(vec![ @@ -701,11 +719,15 @@ impl Engine { } pub fn record_result(&mut self, test_id: &str, passed: bool) -> Result { + self.record_result_with(test_id, passed, None) + } + + pub fn record_result_with(&mut self, test_id: &str, passed: bool, duration_ms: Option) -> Result { let stored = normalize_test_id(test_id); let now = now_secs(); self.store.edit_and_save(|doc| { prune_old_results(doc, now); - append_test_result(doc, test_id, passed, now); + append_test_result_full(doc, test_id, passed, now, duration_ms); })?; Ok(Value::object(vec![ ("test_id".into(), Value::String(stored)), @@ -779,7 +801,7 @@ impl Engine { let content = content_cache.entry(file.clone()).or_insert_with(|| { fs::read_to_string(self.project_dir.join(file)).unwrap_or_default() }); - let deps = extract::extract_deps(file, content); + let deps = extract::extract_deps_in(file, content, Some(&self.project_dir), self.allow_repo_extractor); let resolved = resolved_import_files(&deps, &all_units); let test_shard = shard_owner(file, &shards); for path in &resolved { @@ -842,7 +864,7 @@ impl Engine { fs::read_to_string(self.project_dir.join(rel)).unwrap_or_default() }); let from_shard = shard_owner(rel, &shards); - for dep in extract::extract_deps(rel, content) { + for dep in extract::extract_deps_in(rel, content, Some(&self.project_dir), self.allow_repo_extractor) { if dep.dep_type != "import" && dep.dep_type != "tainted_import" { continue; } @@ -1162,4 +1184,155 @@ impl Engine { } } + pub(super) fn cleanup_orphaned_results(&mut self) -> i64 { + let ids: BTreeSet = self.store.array("test_units").into_iter().filter_map(|u| u.get("id").and_then(|v| v.as_str()).map(|s| s.to_string())).collect(); + if ids.is_empty() { + return 0; + } + let before = self.store.array("test_results").len(); + let kept: Vec = self.store.array("test_results").into_iter().filter(|row| { + row.get("test_id").and_then(|v| v.as_str()).is_some_and(|id| ids.contains(id)) + }).collect(); + let removed = before.saturating_sub(kept.len()) as i64; + if removed > 0 { + self.store.doc.insert("test_results", Value::Array(kept)); + } + removed + } + + /// Copy recorded results out of a legacy SQLite `chisel.db` into the snapshot. + pub fn migrate_legacy(&mut self, db: &std::path::Path) -> Result { + let rows = crate::sqlite_import::read_test_results(db)?; + let mut added = 0i64; + let mut skipped = 0i64; + self.store.edit_and_save(|doc| { + let existing: Vec = doc.get("test_results").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); + for row in &rows { + let id = normalize_test_id(&row.test_id); + let dup = existing.iter().chain(doc.get("test_results").and_then(|v| v.as_array()).into_iter().flatten()).any(|old| { + old.get("test_id").and_then(|v| v.as_str()) == Some(id.as_str()) + && (old.get("recorded_at").and_then(|v| v.as_f64()).unwrap_or(-1.0) - row.recorded_at).abs() < 0.5 + }); + if dup { + skipped += 1; + continue; + } + append_test_result_full(doc, &row.test_id, row.passed, row.recorded_at, row.duration_ms); + added += 1; + } + }).map_err(|e| e.to_string())?; + Ok(Value::object(vec![ + ("imported".into(), Value::Int(added)), + ("skipped".into(), Value::Int(skipped)), + ("source".into(), Value::String(db.display().to_string())), + ("status".into(), Value::String("ok".into())), + ])) + } + +} + +fn unit_churn_from(commits: &[crate::git::Commit], file: &str) -> (i64, i64, i64, i64, String, f64) { + let now = impact::now_unix(); + let mut dates = Vec::new(); + let mut authors = BTreeSet::new(); + let mut ins = 0i64; + let mut del = 0i64; + let mut last = String::new(); + for commit in commits { + if impact::parse_iso_unix(&commit.date).is_none() { + continue; + } + dates.push(commit.date.clone()); + authors.insert(commit.author.clone()); + if last.is_empty() || commit.date > last { + last = commit.date.clone(); + } + for item in &commit.files { + if item.path.is_empty() || item.path == file { + ins += item.insertions; + del += item.deletions; + } + } + } + let score = impact::churn_score(&dates, now); + (dates.len() as i64, authors.len() as i64, ins, del, last, score) +} + +fn backfill_heuristic(test_units: &[Value], edges: &mut Vec, code_units: &[Value]) { + let mut file_of: BTreeMap = BTreeMap::new(); + for unit in test_units { + let id = unit.get("id").and_then(|v| v.as_str()).unwrap_or(""); + let file = unit.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); + if !id.is_empty() { + file_of.insert(id.to_string(), file.to_string()); + } + } + let mut covered: BTreeSet = BTreeSet::new(); + for edge in edges.iter() { + let tid = edge.get("test_id").and_then(|v| v.as_str()).unwrap_or(""); + if let Some(file) = file_of.get(tid) { + covered.insert(file.clone()); + } + } + let mut by_stem: BTreeMap> = BTreeMap::new(); + for code in code_units { + let path = code.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); + let cid = code.get("id").and_then(|v| v.as_str()).unwrap_or(""); + if path.is_empty() || cid.is_empty() || path.contains(".test.") || path.contains(".spec.") { + continue; + } + let base = path.rsplit(['/', '\\']).next().unwrap_or(path); + let base_stem = base.split('.').next().unwrap_or(base).to_ascii_lowercase(); + if base_stem.is_empty() { + continue; + } + by_stem.entry(base_stem).or_default().push((path.to_string(), cid.to_string())); + } + let mut seen = BTreeSet::new(); + for unit in test_units { + let file = unit.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); + let tid = unit.get("id").and_then(|v| v.as_str()).unwrap_or(""); + if file.is_empty() || tid.is_empty() || covered.contains(file) { + continue; + } + let stem = source_stem(file).to_ascii_lowercase(); + let Some(cands) = by_stem.get(&stem) else { continue }; + for (path, cid) in cands { + if path == file { + continue; + } + let key = format!("{tid}|{cid}|heuristic"); + if !seen.insert(key) { + continue; + } + edges.push(Value::object(vec![ + ("test_id".into(), Value::String(tid.into())), + ("code_id".into(), Value::String(cid.clone())), + ("edge_type".into(), Value::String("heuristic".into())), + ("weight".into(), Value::Float(0.2)), + ])); + } + } +} + +fn source_stem(test_file: &str) -> String { + let base = test_file.rsplit(['/', '\\']).next().unwrap_or(test_file); + let mut stem = base.split('.').next().unwrap_or(base).to_string(); + for affix in ["_test", "Test", "_spec", "Spec"] { + if let Some(rest) = stem.strip_suffix(affix) { + if !rest.is_empty() { + stem = rest.to_string(); + break; + } + } + } + for affix in ["test_", "Test"] { + if let Some(rest) = stem.strip_prefix(affix) { + if !rest.is_empty() { + stem = rest.to_string(); + break; + } + } + } + stem } diff --git a/src/engine/mod.rs b/src/engine/mod.rs index a5d495b..e799d3f 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -210,6 +210,12 @@ pub struct Engine { coverage_mode: String, proximity_adjustment: bool, exclude_new_file_boost: bool, + /// Drop test files from risk, gaps, and triage unless the caller opts out. + pub exclude_tests: bool, + /// `suggest_tests` fallback_to_all. Stem fallback still runs when this is false and nothing matched. + pub fallback_to_all: bool, + /// `test_gaps` file scope. Directory scope uses `directory`. + scope_file: Option, /// Parent storage dir when this engine is a shard view. Result rows live there. result_log: Option, /// `(shard name, full shard config)` for overlaying cross edges. `shards` stays empty on a view. @@ -244,6 +250,9 @@ impl Engine { coverage_mode: "line".into(), proximity_adjustment: true, exclude_new_file_boost: false, + exclude_tests: true, + fallback_to_all: false, + scope_file: None, result_log: None, shard_view: None, overlay_applied: false, @@ -321,16 +330,33 @@ pub(crate) fn prune_old_results(doc: &mut Value, now: f64) { } pub(crate) fn append_test_result(doc: &mut Value, test_id: &str, passed: bool, now: f64) { + append_test_result_full(doc, test_id, passed, now, None); +} + +pub(crate) fn append_test_result_full(doc: &mut Value, test_id: &str, passed: bool, now: f64, duration_ms: Option) { if doc.get("test_results").and_then(|v| v.as_array()).is_none() { doc.insert("test_results", Value::Array(Vec::new())); } if let Some(results) = doc.get_mut("test_results").and_then(|v| v.as_array_mut()) { - results.push(Value::object(vec![ + let mut row = vec![ ("test_id".into(), Value::String(normalize_test_id(test_id))), ("passed".into(), Value::Bool(passed)), ("recorded_at".into(), Value::Float(now)), - ])); + ]; + if let Some(ms) = duration_ms { + row.push(("duration_ms".into(), Value::Float(ms))); + } + results.push(Value::object(row)); + } +} + +pub(crate) fn path_in_directory(path: &str, dir: &str) -> bool { + let dir = dir.trim_matches('/').replace('\\', "/"); + if dir.is_empty() || dir == "." { + return true; } + let path = path.replace('\\', "/"); + path == dir || path.starts_with(&format!("{dir}/")) } /// Newest week fail-rate minus the oldest week fail-rate over 28 days. diff --git a/src/engine/query.rs b/src/engine/query.rs index d1a2276..9129800 100644 --- a/src/engine/query.rs +++ b/src/engine/query.rs @@ -44,7 +44,21 @@ impl Engine { if !self.has_data() { return Ok(Self::no_data()); } - let mut files: Vec = self.store.array("churn").into_iter().filter_map(|c| c.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).filter(|p| !looks_like_test(p)).collect(); + let gtest_files: BTreeSet = self.store.array("test_units").into_iter().filter_map(|u| { + let framework = u.get("framework").and_then(|v| v.as_str())?; + if framework == "gtest" { u.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string()) } else { None } + }).collect(); + let mut files: Vec = self.store.array("churn").into_iter().filter_map(|c| c.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).filter(|p| { + if let Some(dir) = &self.directory { + if !path_in_directory(p, dir) { + return false; + } + } + if self.exclude_tests && (looks_like_test(p) || gtest_files.contains(p)) { + return false; + } + true + }).collect(); files.sort(); files.dedup(); let query_min: i64 = store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(1); @@ -242,7 +256,37 @@ impl Engine { } let gap_mode = if self.proximity_adjustment { "proximity_adjusted" } else { self.coverage_mode.as_str() }; meta.insert("coverage_gap_mode", Value::String(gap_mode.into())); - meta.insert("cycles", Value::Array(vec![])); + let scored_paths: Vec = scored.iter().filter_map(|row| row.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).collect(); + let scored_set: BTreeSet = scored_paths.iter().cloned().collect(); + let mut directed: BTreeMap> = BTreeMap::new(); + for edge in &imports { + let conf = edge.get("confidence").and_then(|v| v.as_f64()).unwrap_or(1.0); + if conf < 1.0 { + continue; + } + let from = edge.get("importer_file").and_then(|v| v.as_str()).unwrap_or(""); + let to = edge.get("imported_file").and_then(|v| v.as_str()).unwrap_or(""); + if scored_set.contains(from) && scored_set.contains(to) && from != to { + directed.entry(from.to_string()).or_default().insert(to.to_string()); + } + } + meta.insert("cycles", Value::Array(impact::import_cycles(&scored_paths, &directed))); + let locks = self.active_locks(); + if !locks.is_empty() { + let lock_map: BTreeMap = locks.iter().filter_map(|lock| { + Some((lock.get("file_path")?.as_str()?.to_string(), lock.get("agent_id")?.as_str()?.to_string())) + }).collect(); + let mut claims = 0i64; + for row in &mut scored { + let path = row.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); + if let Some(agent) = lock_map.get(path) { + row.insert("locked_by", Value::String(agent.clone())); + claims += 1; + } + } + meta.insert("active_locks", Value::Int(locks.len() as i64)); + meta.insert("lock_claims", Value::Int(claims)); + } if let Some(bc) = store::meta_get(&self.store.doc, "branch_coupling_commits") { if let Ok(n) = bc.parse::() { meta.insert("branch_coupling_commits", Value::Int(n)); @@ -341,6 +385,14 @@ impl Engine { if !self.has_data() { return Ok(Self::no_data()); } + if self.store.hash_for_file(file).is_empty() && !self.working_tree { + return Ok(Value::object(vec![ + ("status".into(), Value::String("stale_db".into())), + ("file_path".into(), Value::String(file.into())), + ("message".into(), Value::String("File not found in analysis database.".into())), + ("hint".into(), Value::String("Run 'chisel update' first to include new or changed files.".into())), + ])); + } let impacted = self.impacted(&[file.to_string()], &BTreeMap::new(), &BTreeSet::new()); let static_hits = self.static_hits(file); let mut by_id: BTreeMap = BTreeMap::new(); @@ -394,7 +446,8 @@ impl Engine { } result.push(row); } - if result.is_empty() { + let skip_stem = result.is_empty() && self.fallback_to_all && self.working_tree && self.store.hash_for_file(file).is_empty(); + if result.is_empty() && !skip_stem { for (tid, file_path, name, relevance) in self.stem_matches(file) { let (relevance, rate_v, boost_v) = apply_failure(relevance, failure_rates.get(&tid).copied().unwrap_or(0.0)); let mut row = Value::object(vec![ @@ -708,9 +761,7 @@ impl Engine { ])); } self.append_working_tree_stale(&mut stale); - Ok(Value::object(vec![ - ("stale_tests".into(), Value::Array(stale)), - ])) + Ok(Value::Array(stale)) } fn append_working_tree_stale(&self, stale: &mut Vec) { @@ -816,9 +867,38 @@ impl Engine { ("test_results".into(), Value::Int(results.len() as i64)), ("passed".into(), Value::Int(passed)), ("failed".into(), Value::Int(failed)), + ("commit_files".into(), Value::Int(self.store.array("commit_files").len() as i64)), + ("blame_cache".into(), Value::Int(self.store.array("blame").len() as i64)), + ("branch_co_changes".into(), Value::Int(self.store.array("branch_co_changes").len() as i64)), + ("churn_stats".into(), Value::Int(self.store.array("churn").len() as i64)), + ("file_hashes".into(), Value::Int(self.store.array("file_hashes").len() as i64)), ("coupling_threshold".into(), Value::Int(impact::coupling_threshold(self.store.array("commits").len() as i64))), - ("co_change_query_min".into(), Value::String(store::meta_get(&self.store.doc, "co_change_query_min").unwrap_or_else(|| "0".into()))), + ("co_change_query_min".into(), Value::Int(store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(3).max(1))), ]; + let edge_rows = self.store.array("test_edges"); + if !edge_rows.is_empty() { + let mut counts: BTreeMap<&str, i64> = BTreeMap::new(); + for edge in &edge_rows { + let kind = edge.get("edge_type").and_then(|v| v.as_str()).unwrap_or(""); + *counts.entry(kind).or_default() += 1; + } + let total = edge_rows.len() as f64; + let dynamic = counts.get("dynamic_import").copied().unwrap_or(0) + counts.get("eval_import").copied().unwrap_or(0); + pairs.push(("shadow_graph".into(), Value::object(vec![ + ("total_edges".into(), Value::Int(edge_rows.len() as i64)), + ("call_edges".into(), Value::Int(counts.get("call").copied().unwrap_or(0))), + ("import_edges".into(), Value::Int(counts.get("import").copied().unwrap_or(0))), + ("dynamic_import_edges".into(), Value::Int(dynamic)), + ("eval_import_edges".into(), Value::Int(counts.get("eval_import").copied().unwrap_or(0))), + ("tainted_import_edges".into(), Value::Int(counts.get("tainted_import").copied().unwrap_or(0))), + ("unknown_shadow_ratio".into(), Value::Float(crate::json::py_round(dynamic as f64 / total.max(1.0), 4))), + ]))); + } + if let Some(bc) = store::meta_get(&self.store.doc, "branch_coupling_commits") { + if let Ok(n) = bc.parse::() { + pairs.push(("branch_coupling_commits".into(), Value::Int(n))); + } + } if include_results { pairs.push(("recorded_results".into(), Value::Array(results))); } @@ -972,6 +1052,10 @@ impl Engine { sub.coverage_mode = self.coverage_mode.clone(); sub.proximity_adjustment = self.proximity_adjustment; sub.exclude_new_file_boost = self.exclude_new_file_boost; + sub.exclude_tests = self.exclude_tests; + sub.fallback_to_all = self.fallback_to_all; + sub.scope_file = self.scope_file.clone(); + sub.allow_repo_extractor = self.allow_repo_extractor; sub.diff_ref_override = self.diff_ref_override.clone(); sub.directory = self.directory.clone(); sub.git = self.git.fork(); @@ -1036,10 +1120,10 @@ impl Engine { ("message".into(), Value::String("No test edges in the analysis database.".into())), ])); } - Ok(Value::object(vec![("stale_tests".into(), Value::Array(stale))])) + Ok(Value::Array(stale)) } - pub(super) fn aggregate_impact(&self, files: &[String]) -> Result { + pub(super) fn aggregate_impact(&self, files: &[String], functions: &BTreeMap>) -> Result { let mut groups: BTreeMap> = BTreeMap::new(); for file in files { groups.entry(shard_owner(file, &self.shards).to_string()).or_default().push(file.clone()); @@ -1052,7 +1136,7 @@ impl Engine { continue; } any = true; - rows.extend(sub.impacted(&group, &BTreeMap::new(), &BTreeSet::new())); + rows.extend(sub.impacted(&group, functions, &BTreeSet::new())); } if !any { return Ok(Self::no_data()); @@ -1293,7 +1377,24 @@ impl Engine { } pub(super) fn dispatch_inner(&mut self, name: &str, args: &Value) -> Result { - if let Some(field) = first_missing_required(name, args) { + if name == "suggest_tests" { + let has_file = !argument_missing(args, "file_path", "string"); + let has_dir = args.get("directory").and_then(|v| v.as_str()).is_some_and(|s| !s.is_empty()); + if has_file && has_dir { + return Ok(Value::object(vec![ + ("status".into(), Value::String("error".into())), + ("message".into(), Value::String("Provide only one of file_path or directory, not both.".into())), + ])); + } + if !has_file && !has_dir { + return Ok(missing_arg("suggest_tests", "file_path")); + } + if has_file { + if let Some(field) = first_missing_required(name, args) { + return Ok(missing_arg(name, &field)); + } + } + } else if let Some(field) = first_missing_required(name, args) { return Ok(missing_arg(name, &field)); } if let Some(err) = self.unknown_shard(args) { @@ -1323,6 +1424,13 @@ impl Engine { } self.diff_ref_override = args.get("ref").and_then(|v| v.as_str()).map(|s| s.to_string()); self.directory = args.get("directory").and_then(|v| v.as_str()).filter(|s| !s.is_empty() && *s != ".").map(|s| s.trim_end_matches('/').to_string()); + self.exclude_tests = args.get("exclude_tests").and_then(|v| v.as_bool()).unwrap_or(true); + self.fallback_to_all = args.get("fallback_to_all").and_then(|v| v.as_bool()).unwrap_or(false); + self.scope_file = if name == "test_gaps" { + args.get("file_path").and_then(|v| v.as_str()).filter(|s| !s.is_empty()).map(|s| s.to_string()) + } else { + None + }; let auto = self.auto_plan(name, args); if matches!(auto, AutoPlan::Run) { let _ = self.update(); @@ -1342,23 +1450,36 @@ impl Engine { "diff_impact" => self.diff_impact(), "suggest_tests" => { let file = args.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); - self.suggest_tests(file) + if file.is_empty() { + let dir = self.directory.clone().unwrap_or_default(); + self.suggest_directory(&dir) + } else { + self.suggest_tests(file) + } } "stale_tests" => self.stale_tests(), "stats" => self.stats_with(args.get("include_results").and_then(|v| v.as_bool()).unwrap_or(false)), "record_result" => { let id = args.get("test_id").and_then(|v| v.as_str()).unwrap_or(""); let passed = args.get("passed").and_then(|v| v.as_bool()).unwrap_or(false); - self.record_result(id, passed) + let duration = args.get("duration_ms").and_then(|v| v.as_f64()); + self.record_result_with(id, passed, duration) } "impact" => { let files: Vec = args.get("files").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect()).unwrap_or_default(); + let names: Vec = args.get("functions").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).filter(|s| !s.is_empty()).collect()).unwrap_or_default(); + let mut functions: BTreeMap> = BTreeMap::new(); + if !names.is_empty() { + for file in &files { + functions.insert(file.clone(), names.clone()); + } + } if !self.shards.is_empty() { - self.aggregate_impact(&files) + self.aggregate_impact(&files, &functions) } else { self.store.reload()?; if !self.has_data() { return Ok(Self::no_data()); } - Ok(Value::Array(self.impacted(&files, &BTreeMap::new(), &BTreeSet::new()))) + Ok(Value::Array(self.impacted(&files, &functions, &BTreeSet::new()))) } } "test_gaps" => self.test_gaps(), @@ -1425,10 +1546,23 @@ impl Engine { pub(super) fn gap_rows(&self) -> Vec { let edges = self.store.array("test_edges"); let churn = self.store.array("churn"); + let gtest_files: BTreeSet = self.store.array("test_units").into_iter().filter_map(|u| { + if u.get("framework").and_then(|v| v.as_str()) == Some("gtest") { + u.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string()) + } else { + None + } + }).collect(); let mut gaps = Vec::new(); for u in self.store.array("code_units") { let path = u.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); - if looks_like_test(path) || is_test_unit_row(&u) { continue; } + if let Some(file) = &self.scope_file { + if path != file.as_str() { continue; } + } else if let Some(dir) = &self.directory { + if !path_in_directory(path, dir) { continue; } + } + if is_test_unit_row(&u) && self.exclude_tests { continue; } + if self.exclude_tests && (looks_like_test(path) || gtest_files.contains(path)) { continue; } let id = u.get("id").and_then(|v| v.as_str()).unwrap_or(""); if edges.iter().any(|e| e.get("code_id").and_then(|v| v.as_str()) == Some(id)) { continue; @@ -1596,7 +1730,16 @@ impl Engine { if !self.has_data() { return Ok(Self::no_data()); } match name { "churn" => { - let rows: Vec = self.store.array("churn").into_iter().filter(|c| c.get("file_path").and_then(|v| v.as_str()) == Some(file)).collect(); + let unit = args.get("unit_name").and_then(|v| v.as_str()).filter(|s| !s.is_empty()); + let mut rows: Vec = self.store.array("churn").into_iter().filter(|c| c.get("file_path").and_then(|v| v.as_str()) == Some(file)).collect(); + if let Some(unit) = unit { + rows.retain(|c| c.get("unit_name").and_then(|v| v.as_str()).unwrap_or("") == unit); + } else { + let file_rows: Vec = rows.iter().filter(|c| c.get("unit_name").and_then(|v| v.as_str()).unwrap_or("").is_empty()).cloned().collect(); + if !file_rows.is_empty() { + rows = file_rows; + } + } Ok(Value::Array(rows)) } "history" => Ok(Value::Array(self.history_rows(file))), @@ -1764,12 +1907,26 @@ impl Engine { if a == file && b != file && self.project_dir.join(b).exists() { neighbors.insert(b.to_string()); } if b == file && a != file && self.project_dir.join(a).exists() { neighbors.insert(a.to_string()); } } + let untracked = self.working_tree && self.store.hash_for_file(file).is_empty(); + if untracked && neighbors.is_empty() { + if let Ok(text) = fs::read_to_string(self.project_dir.join(file)) { + for dep in extract::extract_deps_in(file, &text, Some(&self.project_dir), self.allow_repo_extractor) { + if dep.dep_type == "import" { + if let Some(module) = dep.module_path { + if neighbors.len() < 20 { + neighbors.insert(module); + } + } + } + } + } + } let co_len = partners.len() as f64; let imp_len = neighbors.len() as f64; let co_norm = (co_len / impact::COCHANGE_CAP).min(1.0); let import_norm = (imp_len / impact::IMPORT_CAP).min(1.0); let effective_coupling = co_norm.max(import_norm).max(0.5 * co_norm + 0.5 * import_norm); - Value::object(vec![ + let mut body = vec![ ("co_change_partners".into(), Value::Array(partners)), ("import_partners".into(), Value::Array(neighbors.into_iter().map(|f| Value::object(vec![("file".into(), Value::String(f))])).collect())), ("co_change_breadth".into(), Value::Int(co_len as i64)), @@ -1777,7 +1934,44 @@ impl Engine { ("cochange_coupling".into(), Value::Float(crate::json::py_round(co_norm, 4))), ("import_coupling".into(), Value::Float(crate::json::py_round(import_norm, 4))), ("effective_coupling".into(), Value::Float(crate::json::py_round(effective_coupling, 4))), - ]) + ]; + if untracked { + body.push(("note".into(), Value::String("working_tree: import_partners from on-disk scan (not yet in DB import graph)".into()))); + } + Value::object(body) + } + + pub(super) fn suggest_directory(&mut self, dir: &str) -> Result { + if !self.shards.is_empty() { + let mut merged: BTreeMap = BTreeMap::new(); + for name in self.snapshot_names() { + let mut sub = self.open_shard(&name)?; + let part = sub.suggest_directory(dir)?; + if let Some(pairs) = part.as_object() { + for (k, v) in pairs { + merged.insert(k.clone(), v.clone()); + } + } + } + return Ok(Value::object(merged.into_iter().collect())); + } + self.store.reload()?; + if !self.has_data() { + return Ok(Self::no_data()); + } + let mut files: Vec = self.store.array("file_hashes").into_iter().filter_map(|h| h.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).filter(|p| path_in_directory(p, dir) && !looks_like_test(p)).collect(); + files.sort(); + files.dedup(); + let mut pairs = Vec::new(); + for file in files { + let suggested = self.suggest_tests(&file)?; + if let Some(rows) = suggested.as_array() { + if !rows.is_empty() { + pairs.push((file, Value::Array(rows.to_vec()))); + } + } + } + Ok(Value::object(pairs)) } } diff --git a/src/extract/brace.rs b/src/extract/brace.rs index 845c37f..a7a1d1a 100644 --- a/src/extract/brace.rs +++ b/src/extract/brace.rs @@ -117,6 +117,11 @@ fn decl_on_line(t: &str, lang: &str) -> Option<(String, String)> { return Some(found); } } + if matches!(lang, "c" | "cpp") { + if let Some(name) = super::framework::cpp_macro_name(t) { + return Some((name, "test_case".into())); + } + } if matches!(lang, "javascript" | "typescript" | "php" | "kotlin" | "dart" | "swift" | "java" | "csharp" | "c" | "cpp") { if let Some(name) = after_keyword_fn(t, &["function ", "func ", "fun "]) { return Some((name, "function".into())); @@ -337,10 +342,14 @@ fn method_name(t: &str) -> Option { } let name = parts.last()?.trim_end_matches(')'); let name = name.trim_matches(|c: char| !c.is_ascii_alphanumeric() && c != '_'); - if name.is_empty() || is_modifier(name) || matches!(name, "if" | "for" | "while" | "switch" | "return" | "new") { + if name.is_empty() || is_modifier(name) || matches!(name, "if" | "for" | "while" | "switch" | "return" | "new" | "require" | "include") { + return None; + } + // `const x = require(...)` is a call, not a declaration. + let before = parts[parts.len() - 2]; + if before == "=" || before.ends_with('=') { return None; } - // The token before the name should look like a type, not an operator. Some(name.to_string()) } diff --git a/src/extract/deps.rs b/src/extract/deps.rs index c5be15d..0dcdaf7 100644 --- a/src/extract/deps.rs +++ b/src/extract/deps.rs @@ -9,14 +9,110 @@ pub struct Dep { } pub fn extract_deps(path: &str, content: &str) -> Vec { + extract_deps_in(path, content, None, false) +} + +/// `project` lets `.chisel/extensions.toml` name a dependency extractor. +/// `CHISEL_DEP_EXTRACTOR` still wins, and does not need `allow_repo`. +pub fn extract_deps_in(path: &str, content: &str, project: Option<&std::path::Path>, allow_repo: bool) -> Vec { + if let Some(custom) = super::custom_deps(path, content, project, allow_repo) { + return custom; + } match detect_language(path) { Some("python") => python_deps(content), Some("javascript" | "typescript") => js_deps(content), Some("go") => go_deps(content), + Some("java" | "kotlin") => import_lines(content, &["import "], &["."], false), + Some("csharp") => import_lines(content, &["using "], &["."], false), + Some("c" | "cpp") => include_lines(content), + Some("ruby") => ruby_requires(content), + Some("php") => import_lines(content, &["use "], &["\\"], false), + Some("swift") => import_lines(content, &["import "], &["."], false), + Some("dart") => import_lines(content, &["import "], &["/"], false), _ => name_calls(content), } } +fn import_lines(content: &str, prefixes: &[&str], seps: &[&str], with_calls: bool) -> Vec { + let mut deps = Vec::new(); + for line in content.lines() { + let t = line.trim().trim_end_matches(';'); + let mut rest = None; + for prefix in prefixes { + if let Some(r) = t.strip_prefix(prefix) { + rest = Some(r.trim()); + break; + } + } + let Some(rest) = rest else { continue }; + let rest = rest.trim_start_matches("static "); + let path = rest.split_whitespace().next().unwrap_or("").trim_matches(|c| c == '"' || c == '\'' || c == ';'); + if path.is_empty() || path == "*" { + continue; + } + let mut name = path; + for sep in seps { + if let Some(tail) = path.rsplit_once(sep) { + name = tail.1; + break; + } + } + if name.is_empty() || name == "*" { + continue; + } + deps.push(Dep { + name: name.to_string(), + dep_type: "import".into(), + module_path: Some(path.to_string()), + confidence: 1.0, + }); + } + if with_calls { + deps.extend(name_calls(content)); + } + dedupe(deps) +} + +fn include_lines(content: &str) -> Vec { + let mut deps = Vec::new(); + for line in content.lines() { + let t = line.trim(); + let Some(rest) = t.strip_prefix("#include") else { continue }; + let rest = rest.trim(); + let path = rest.trim_matches(|c| c == '<' || c == '>' || c == '"' || c == ' '); + if path.is_empty() { + continue; + } + let base = path.rsplit(['/', '\\']).next().unwrap_or(path); + let name = base.split('.').next().unwrap_or(base); + deps.push(Dep { + name: name.to_string(), + dep_type: "import".into(), + module_path: Some(path.to_string()), + confidence: 1.0, + }); + } + dedupe(deps) +} + +fn ruby_requires(content: &str) -> Vec { + let mut deps = Vec::new(); + for line in content.lines() { + let t = line.trim(); + let rest = t.strip_prefix("require_relative ").or_else(|| t.strip_prefix("require ")); + let Some(rest) = rest else { continue }; + let Some(path) = quoted_token(rest) else { continue }; + let name = path.rsplit(['/', '\\']).next().unwrap_or(&path); + deps.push(Dep { + name: name.to_string(), + dep_type: "import".into(), + module_path: Some(path), + confidence: 1.0, + }); + } + dedupe(deps) +} + fn python_deps(content: &str) -> Vec { let mut deps = Vec::new(); for line in content.lines() { @@ -533,8 +629,14 @@ pub fn proximity_weight(test_path: &str, code_path: &str) -> f64 { pub fn is_test_name(name: &str, framework: &str) -> bool { let bare = name.rsplit('.').next().unwrap_or(name); match framework { - "pytest" => bare.starts_with("test_"), + "pytest" => bare.starts_with("test_") || bare.starts_with("Test"), + "jest" | "playwright" => matches!(bare, "describe" | "it" | "test") || bare.starts_with("test"), "go" => bare.starts_with("Test") || bare.starts_with("Benchmark"), + "rust" => bare.starts_with("test_"), + "csharp_test" => bare.starts_with("Test") || bare.ends_with("Test") || bare.ends_with("Tests"), + "gtest" => true, + "junit" | "xctest" | "minitest" | "phpunit" | "dart_test" => bare.starts_with("test") || bare.starts_with("Test"), + "rspec" => matches!(bare, "describe" | "it" | "context") || bare.starts_with("test"), _ => bare.to_ascii_lowercase().starts_with("test"), } } diff --git a/src/extract/framework.rs b/src/extract/framework.rs new file mode 100644 index 0000000..7eeaf7f --- /dev/null +++ b/src/extract/framework.rs @@ -0,0 +1,291 @@ +//! Test-framework labels and annotation names. Filename rules stay in `looks_like_test`. + +use std::collections::BTreeSet; + +/// Framework label for a path, using file content when the name is ambiguous. +/// `None` means the built-in filename rules did not mark it and the content has no test macro. +pub fn framework_for(path: &str, content: &str) -> Option<&'static str> { + let name = path.rsplit(['/', '\\']).next().unwrap_or(path); + if (name.starts_with("test_") && name.ends_with(".py")) || name.ends_with("_test.py") { + return Some("pytest"); + } + if name.ends_with("_test.go") { + return Some("go"); + } + if name.ends_with(".test.js") || name.ends_with(".test.jsx") || name.ends_with(".test.ts") || name.ends_with(".test.tsx") { + return Some("jest"); + } + if name.ends_with(".spec.js") || name.ends_with(".spec.jsx") || name.ends_with(".spec.ts") || name.ends_with(".spec.tsx") { + return Some(if content_has_playwright(content) { "playwright" } else { "jest" }); + } + if name.ends_with("Test.cs") || name.ends_with("Tests.cs") { + return Some("csharp_test"); + } + if name.ends_with("Test.java") || (name.starts_with("Test") && name.ends_with(".java")) || name.ends_with("Test.kt") || name.ends_with("Test.kts") { + return Some("junit"); + } + if name.ends_with("Test.swift") || name.ends_with("Tests.swift") { + return Some("xctest"); + } + if name.ends_with("Test.php") { + return Some("phpunit"); + } + if name.ends_with("_spec.rb") { + return Some("rspec"); + } + if name.starts_with("test_") && name.ends_with(".rb") { + return Some("minitest"); + } + if name.ends_with("_test.dart") { + return Some("dart_test"); + } + if name.ends_with("_test.rs") || name.ends_with(".rs") && path_under_tests(path) { + return Some("rust"); + } + if name.ends_with(".rs") && content_has_rust_test(content) { + return Some("rust"); + } + if is_c_family(name) && content_has_cpp_test(content) { + return Some("gtest"); + } + None +} + +fn path_under_tests(path: &str) -> bool { + path.replace('\\', "/").split('/').next() == Some("tests") +} + +fn is_c_family(name: &str) -> bool { + name.ends_with(".c") + || name.ends_with(".h") + || name.ends_with(".cc") + || name.ends_with(".cpp") + || name.ends_with(".cxx") + || name.ends_with(".hpp") + || name.ends_with(".hxx") +} + +pub fn content_has_playwright(content: &str) -> bool { + content.contains("playwright") || content.contains("@playwright") +} + +pub fn content_has_rust_test(content: &str) -> bool { + content.contains("#[test]") || content.contains("#[tokio::test]") || content.contains("#[rstest]") || content.contains("#[cfg(test)]") +} + +pub fn content_has_cpp_test(content: &str) -> bool { + content.contains("TEST(") || content.contains("TEST_F(") || content.contains("TEST_CASE(") || content.contains("BOOST_AUTO_TEST_CASE(") +} + +/// `TEST(Suite, Name)` and the Catch2 / Boost forms become one test unit name. +pub fn cpp_macro_name(line: &str) -> Option { + let t = line.trim(); + if let Some(rest) = macro_rest(t, "BOOST_AUTO_TEST_CASE") { + return ident_arg(rest); + } + if let Some(rest) = macro_rest(t, "BOOST_FIXTURE_TEST_CASE") { + return ident_arg(rest); + } + if let Some(rest) = macro_rest(t, "TEST_CASE") { + return quoted_or_ident(rest); + } + if let Some(rest) = macro_rest(t, "SCENARIO") { + return quoted_or_ident(rest); + } + if let Some(rest) = macro_rest(t, "TEST_F") { + return suite_case(rest); + } + if let Some(rest) = macro_rest(t, "TEST_P") { + return suite_case(rest); + } + if let Some(rest) = macro_rest(t, "TEST") { + return suite_case(rest); + } + None +} + +fn macro_rest<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let rest = line.strip_prefix(name)?; + let rest = rest.trim_start(); + rest.strip_prefix('(') +} + +fn ident_arg(rest: &str) -> Option { + let name: String = rest.trim_start().chars().take_while(|c| c.is_ascii_alphanumeric() || *c == '_').collect(); + if name.is_empty() { None } else { Some(name) } +} + +fn quoted_or_ident(rest: &str) -> Option { + let t = rest.trim_start(); + if t.starts_with('"') || t.starts_with('\'') { + let q = t.chars().next()?; + let body = &t[q.len_utf8()..]; + let end = body.find(q)?; + let name = body[..end].trim(); + if name.is_empty() { None } else { Some(name.to_string()) } + } else { + ident_arg(t) + } +} + +fn suite_case(rest: &str) -> Option { + let suite = ident_arg(rest)?; + let after = rest.trim_start()[suite.len()..].trim_start(); + let after = after.strip_prefix(',')?.trim_start(); + let name = ident_arg(after)?; + Some(format!("{suite}.{name}")) +} + +/// Names marked with `@Test`, `[Fact]` / `[Theory]` / `[Test]` / `[TestMethod]`, `@Test`, or `#[test]`. +pub fn annotated_names(content: &str, framework: &str) -> BTreeSet { + match framework { + "junit" => java_annotated(content), + "csharp_test" => csharp_annotated(content), + "xctest" => swift_annotated(content), + "rust" => rust_annotated(content), + _ => BTreeSet::new(), + } +} + +fn java_annotated(content: &str) -> BTreeSet { + let lines: Vec<&str> = content.lines().collect(); + let mut names = BTreeSet::new(); + let mut i = 0; + while i < lines.len() { + if java_test_attr(lines[i]) { + if let Some(name) = method_after_attrs(&lines[i..]) { + names.insert(name); + } + } + i += 1; + } + names +} + +fn java_test_attr(line: &str) -> bool { + let t = line.trim(); + t.contains("@Test") || t.contains("@ParameterizedTest") || t.contains("@RepeatedTest") +} + +fn csharp_annotated(content: &str) -> BTreeSet { + let lines: Vec<&str> = content.lines().collect(); + let mut names = BTreeSet::new(); + for (i, line) in lines.iter().enumerate() { + if csharp_test_attr(line) { + if let Some(name) = method_after_attrs(&lines[i..]) { + names.insert(name); + } + } + } + names +} + +fn csharp_test_attr(line: &str) -> bool { + let t = line.trim(); + for attr in ["[Fact", "[Theory", "[TestMethod", "[Test"] { + if let Some(rest) = t.strip_prefix(attr) { + let next = rest.chars().next().unwrap_or(']'); + if next == ']' || next == '(' || next.is_whitespace() { + return true; + } + } + } + false +} + +fn swift_annotated(content: &str) -> BTreeSet { + let lines: Vec<&str> = content.lines().collect(); + let mut names = BTreeSet::new(); + for (i, line) in lines.iter().enumerate() { + let t = line.trim(); + if t.starts_with("@Test") { + if let Some(name) = swift_func_from(t) { + names.insert(name); + continue; + } + if let Some(name) = method_after_attrs(&lines[i..]) { + names.insert(name); + } + } + } + names +} + +fn swift_func_from(line: &str) -> Option { + let idx = line.find("func ")?; + let rest = line[idx + 5..].trim_start(); + let name: String = rest.chars().take_while(|c| c.is_ascii_alphanumeric() || *c == '_').collect(); + if name.is_empty() { None } else { Some(name) } +} + +fn rust_annotated(content: &str) -> BTreeSet { + let lines: Vec<&str> = content.lines().collect(); + let mut names = BTreeSet::new(); + for (i, line) in lines.iter().enumerate() { + let t = line.trim(); + if t.starts_with("#[test]") || t.starts_with("#[tokio::test]") || t.starts_with("#[rstest]") || t.starts_with("#[test_case") { + for next in lines.iter().skip(i).take(6) { + let n = next.trim().trim_start_matches("async ").trim_start(); + if let Some(rest) = n.strip_prefix("fn ") { + let name: String = rest.trim_start().chars().take_while(|c| c.is_ascii_alphanumeric() || *c == '_').collect(); + if !name.is_empty() { + names.insert(name); + } + break; + } + } + } + } + names +} + +/// Walk from an attribute line to the next method signature and take its name. +fn method_after_attrs(lines: &[&str]) -> Option { + for line in lines.iter().take(8) { + let t = line.trim(); + if t.is_empty() || is_attr_line(t) { + continue; + } + if let Some(name) = signature_name(t) { + return Some(name); + } + } + None +} + +fn is_attr_line(t: &str) -> bool { + t.starts_with('@') || t.starts_with('[') || t.starts_with('#') +} + +fn signature_name(line: &str) -> Option { + let mut t = line.trim(); + if let Some(idx) = t.find("func ") { + return swift_func_from(&t[idx..]); + } + if let Some(rest) = t.strip_prefix("fn ").or_else(|| t.strip_prefix("fun ")) { + t = rest; + } + let paren = t.find('(')?; + let head = t[..paren].trim(); + if head.contains('=') { + return None; + } + let mut parts: Vec<&str> = head.split_whitespace().collect(); + while parts.len() > 1 && is_sig_modifier(parts[0]) { + parts.remove(0); + } + let name = parts.last()?.trim_matches(|c: char| !c.is_ascii_alphanumeric() && c != '_'); + if name.is_empty() || is_sig_modifier(name) { + None + } else { + Some(name.to_string()) + } +} + +fn is_sig_modifier(w: &str) -> bool { + matches!( + w, + "public" | "private" | "protected" | "internal" | "static" | "final" | "abstract" + | "async" | "override" | "virtual" | "open" | "func" | "fun" | "fn" | "void" + ) +} diff --git a/src/extract/mod.rs b/src/extract/mod.rs index 8901dae..a006314 100644 --- a/src/extract/mod.rs +++ b/src/extract/mod.rs @@ -3,9 +3,11 @@ mod brace; mod deps; +mod framework; mod python; -pub use deps::{extract_deps, Dep, is_test_name, matches_import_path, matches_relative_import, proximity_weight}; +pub use deps::{extract_deps, extract_deps_in, Dep, is_test_name, matches_import_path, matches_relative_import, proximity_weight}; +pub use framework::{annotated_names, content_has_cpp_test, cpp_macro_name, framework_for}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CodeUnit { @@ -162,7 +164,7 @@ fn configured_extractor(project: Option<&std::path::Path>, allow_repo_extractor: Some(canon.to_string_lossy().into_owned()) } -pub(crate) fn extractor_from_toml(text: &str) -> Option { +pub(crate) fn toml_value(text: &str, want: &str) -> Option { for line in text.lines() { let line = line.trim(); if line.is_empty() || line.starts_with('#') || line.starts_with('[') { @@ -171,7 +173,7 @@ pub(crate) fn extractor_from_toml(text: &str) -> Option { let Some((key, rest)) = line.split_once('=') else { continue; }; - if key.trim() != "extractor" { + if key.trim() != want { continue; } let value = rest.trim().trim_matches('"').trim_matches('\''); @@ -183,6 +185,104 @@ pub(crate) fn extractor_from_toml(text: &str) -> Option { None } +pub(crate) fn extractor_from_toml(text: &str) -> Option { + toml_value(text, "extractor") +} + +pub(crate) fn dep_extractor_from_toml(text: &str) -> Option { + toml_value(text, "dep_extractor") +} + +/// Custom dependency extractor. Replaces `register_dep_extractor` and `CHISEL_BOOTSTRAP`. +/// Stdin is `{"path","language","content"}`. Stdout is a JSON list of `{name,dep_type,module_path,confidence}`. +pub(crate) fn custom_deps(path: &str, content: &str, project: Option<&std::path::Path>, allow_repo: bool) -> Option> { + let bin = configured_dep_extractor(project, allow_repo)?; + let lang = detect_language(path).unwrap_or(""); + match run_dep_extractor(&bin, path, lang, content) { + Ok(rows) => Some(rows), + Err(err) => { + note_extractor_warning(format!("CHISEL_DEP_EXTRACTOR failed for {path}: {err}")); + None + } + } +} + +fn configured_dep_extractor(project: Option<&std::path::Path>, allow_repo: bool) -> Option { + if let Ok(bin) = std::env::var("CHISEL_DEP_EXTRACTOR") { + let bin = bin.trim(); + if !bin.is_empty() { + return Some(bin.to_string()); + } + } + if let Ok(boot) = std::env::var("CHISEL_BOOTSTRAP") { + let boot = boot.trim(); + if !boot.is_empty() { + let path = std::path::Path::new(boot); + if path.exists() { + return Some(boot.to_string()); + } + note_once("CHISEL_BOOTSTRAP is a Python import path. Set CHISEL_DEP_EXTRACTOR to an executable instead."); + } + } + if !allow_repo { + return None; + } + let project = project?; + let text = std::fs::read_to_string(project.join(".chisel").join("extensions.toml")).ok()?; + let raw = dep_extractor_from_toml(&text)?; + let path = std::path::PathBuf::from(&raw); + if path.is_absolute() || raw.split(['/', '\\']).any(|part| part == "..") { + note_once("repo dep extractor path must stay inside the project"); + return None; + } + let resolved = project.join(path); + let root = std::fs::canonicalize(project).ok()?; + let canon = std::fs::canonicalize(&resolved).ok()?; + if !canon.starts_with(&root) { + note_once("repo dep extractor path must stay inside the project"); + return None; + } + Some(canon.to_string_lossy().into_owned()) +} + +fn run_dep_extractor(bin: &str, file_path: &str, lang: &str, content: &str) -> Result, String> { + let payload = crate::json::Value::object(vec![ + ("path".into(), crate::json::Value::String(file_path.into())), + ("language".into(), crate::json::Value::String(lang.into())), + ("content".into(), crate::json::Value::String(content.into())), + ]); + let body = crate::json::to_string(&payload).map_err(|e| e.to_string())?; + let mut child = std::process::Command::new(bin) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|e| format!("spawn: {e}"))?; + if let Some(mut stdin) = child.stdin.take() { + use std::io::Write; + stdin.write_all(body.as_bytes()).map_err(|e| format!("stdin: {e}"))?; + } + let output = child.wait_with_output().map_err(|e| format!("wait: {e}"))?; + if !output.status.success() { + return Err(format!("exit {}", output.status.code().unwrap_or(-1))); + } + let text = String::from_utf8_lossy(&output.stdout); + let parsed = crate::json::parse(text.trim()).map_err(|e| format!("bad JSON: {e}"))?; + let rows = parsed.as_array().ok_or_else(|| "bad JSON: expected a dependency array".to_string())?; + let mut deps_out = Vec::new(); + for row in rows { + let name = row.get("name").and_then(|v| v.as_str()).unwrap_or("").to_string(); + if name.is_empty() { + continue; + } + let dep_type = row.get("dep_type").and_then(|v| v.as_str()).unwrap_or("import").to_string(); + let module_path = row.get("module_path").and_then(|v| v.as_str()).map(|s| s.to_string()); + let confidence = row.get("confidence").and_then(|v| v.as_f64()).unwrap_or(1.0); + deps_out.push(deps::Dep { name, dep_type, module_path, confidence }); + } + Ok(deps_out) +} + fn run_extractor(bin: &str, file_path: &str, lang: &str) -> Result, String> { let payload = crate::json::Value::object(vec![ ("path".into(), crate::json::Value::String(file_path.into())), diff --git a/src/git.rs b/src/git.rs index 0bdbdc8..6c1970d 100644 --- a/src/git.rs +++ b/src/git.rs @@ -195,6 +195,9 @@ pub fn parse_log_output(raw: &str) -> Vec { message: parts[4].into(), files: Vec::new(), }; + let mut diff_ins = 0i64; + let mut diff_del = 0i64; + let mut has_numstat = false; for line in lines { let line = line.trim_end(); if line.is_empty() { @@ -206,13 +209,26 @@ pub fn parse_log_output(raw: &str) -> Vec { let ins_ok = ins == "-" || ins.chars().all(|c| c.is_ascii_digit()); let del_ok = del == "-" || del.chars().all(|c| c.is_ascii_digit()); if ins_ok && del_ok { + has_numstat = true; commit.files.push(GitFile { path: path.into(), insertions: if ins == "-" { 0 } else { ins.parse().unwrap_or(0) }, deletions: if del == "-" { 0 } else { del.parse().unwrap_or(0) }, }); + continue; } } + if !has_numstat { + let trimmed = line.trim(); + if trimmed.starts_with('+') && !trimmed.starts_with("+++") { + diff_ins += 1; + } else if trimmed.starts_with('-') && !trimmed.starts_with("---") { + diff_del += 1; + } + } + } + if !has_numstat && (diff_ins > 0 || diff_del > 0) { + commit.files.push(GitFile { path: String::new(), insertions: diff_ins, deletions: diff_del }); } commits.push(commit); } diff --git a/src/http.rs b/src/http.rs index cf8ce36..09d838a 100644 --- a/src/http.rs +++ b/src/http.rs @@ -153,7 +153,13 @@ fn route(method: &str, path: &str, body: &[u8], engine: &Arc>) -> return match eng.dispatch(tool, args) { Ok(result) => { let missing = missing_required_result(tool, &result); - let steps = compute_next_steps(tool, &result); + let mut steps = compute_next_steps(tool, &result); + if tool == "suggest_tests" { + let file = args.get("file_path").and_then(|v| v.as_str()); + if let Some(warning) = crate::next_steps::eval_warning(eng.project_dir.as_path(), file) { + steps.push(warning); + } + } let mut payload = vec![("result".into(), result)]; if !steps.is_empty() { payload.push(("next_steps".into(), Value::Array(steps))); diff --git a/src/impact.rs b/src/impact.rs index 9646486..5b2e57c 100644 --- a/src/impact.rs +++ b/src/impact.rs @@ -449,3 +449,88 @@ pub fn hops_to_tested(files: &BTreeSet, tested: &BTreeSet, neigh } dist } + +struct Tarjan<'a> { + files: &'a [String], + outgoing: &'a BTreeMap>, + index_of: &'a BTreeMap<&'a str, usize>, + index: Vec>, + low: Vec, + on_stack: Vec, + stack: Vec, + next: usize, + sccs: Vec>, +} + +impl Tarjan<'_> { + fn strong(&mut self, v: usize) { + self.index[v] = Some(self.next); + self.low[v] = self.next; + self.next += 1; + self.stack.push(v); + self.on_stack[v] = true; + if let Some(nei) = self.outgoing.get(&self.files[v]) { + let names: Vec = nei.iter().cloned().collect(); + for wname in names { + let Some(&w) = self.index_of.get(wname.as_str()) else { continue }; + if self.index[w].is_none() { + self.strong(w); + self.low[v] = self.low[v].min(self.low[w]); + } else if self.on_stack[w] { + self.low[v] = self.low[v].min(self.index[w].unwrap_or(0)); + } + } + } + if self.low[v] == self.index[v].unwrap_or(0) { + let mut comp = Vec::new(); + while let Some(w) = self.stack.pop() { + self.on_stack[w] = false; + comp.push(w); + if w == v { + break; + } + } + self.sccs.push(comp); + } + } +} + +/// Tarjan strongly connected components on a directed import graph. +/// Returns the three largest cycles, files sorted inside each cycle. +pub fn import_cycles(files: &[String], outgoing: &BTreeMap>) -> Vec { + let index_of: BTreeMap<&str, usize> = files.iter().enumerate().map(|(i, f)| (f.as_str(), i)).collect(); + let n = files.len(); + let mut state = Tarjan { + files, + outgoing, + index_of: &index_of, + index: vec![None; n], + low: vec![0usize; n], + on_stack: vec![false; n], + stack: Vec::new(), + next: 0, + sccs: Vec::new(), + }; + for v in 0..n { + if state.index[v].is_none() { + state.strong(v); + } + } + let sccs = state.sccs; + let mut cycles: Vec = sccs.into_iter().filter(|c| c.len() > 1).map(|comp| { + let mut names: Vec = comp.into_iter().map(|i| files[i].clone()).collect(); + names.sort(); + let len = names.len() as i64; + Value::object(vec![ + ("files".into(), Value::Array(names.into_iter().map(Value::String).collect())), + ("length".into(), Value::Int(len)), + ]) + }).collect(); + cycles.sort_by(|a, b| { + let la = a.get("length").and_then(|v| v.as_i64()).unwrap_or(0); + let lb = b.get("length").and_then(|v| v.as_i64()).unwrap_or(0); + lb.cmp(&la) + }); + cycles.truncate(3); + cycles +} diff --git a/src/lib.rs b/src/lib.rs index b8db043..ee6ec84 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ //! Chisel: test impact analysis for coding agents. Zero crates. pub mod cli; +pub mod cli_text; pub mod engine; pub mod extract; pub mod fswatch; @@ -13,7 +14,9 @@ pub mod json; pub mod lock; pub mod mcp; pub mod next_steps; +pub mod run_cmd; pub mod schemas; +pub mod sqlite_import; pub mod store; pub use engine::Engine; diff --git a/src/mcp.rs b/src/mcp.rs index 0707f81..d354915 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -1,4 +1,5 @@ -//! Stdio MCP framing: Content-Length header, then one JSON-RPC body. +//! Stdio MCP framing. Standard clients send one JSON-RPC object per line. +//! Content-Length framing still works for clients that send it. //! Implements initialize, tools/list, and tools/call. use crate::engine::Engine; @@ -15,7 +16,7 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { let stdout = io::stdout(); let mut output = stdout.lock(); loop { - let body = match read_message(&mut input) { + let (body, newline) = match read_incoming(&mut input) { Ok(Some(body)) => body, Ok(None) => break, Err(err) => { @@ -27,7 +28,7 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { ("message".into(), Value::String(err.to_string())), ])), ]); - write_message(&mut output, &json::to_string(&err_body).unwrap_or_default()).map_err(|e| e.to_string())?; + write_message(&mut output, &json::to_string(&err_body).unwrap_or_default(), true).map_err(|e| e.to_string())?; if err.kind() == io::ErrorKind::InvalidData { continue; } @@ -43,7 +44,7 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { ("message".into(), Value::String("parse error".into())), ])), ]); - write_message(&mut output, &json::to_string(&err).unwrap_or_default()).map_err(|e| e.to_string())?; + write_message(&mut output, &json::to_string(&err).unwrap_or_default(), newline).map_err(|e| e.to_string())?; continue; }; let method = msg.get("method").and_then(|v| v.as_str()).unwrap_or(""); @@ -75,7 +76,13 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { let args = params.and_then(|p| p.get("arguments")).unwrap_or(&empty); match engine.dispatch(name, args) { Ok(result) => { - let steps = compute_next_steps(name, &result); + let mut steps = compute_next_steps(name, &result); + if name == "suggest_tests" { + let file = args.get("file_path").and_then(|v| v.as_str()); + if let Some(warning) = crate::next_steps::eval_warning(engine.project_dir.as_path(), file) { + steps.push(warning); + } + } let mut payload = vec![("result".into(), result)]; if !steps.is_empty() { payload.push(("next_steps".into(), Value::Array(steps))); @@ -111,7 +118,7 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { ("message".into(), Value::String("method not found".into())), ])), ]); - write_message(&mut output, &json::to_string(&err).unwrap_or_default()).map_err(|e| e.to_string())?; + write_message(&mut output, &json::to_string(&err).unwrap_or_default(), newline).map_err(|e| e.to_string())?; } continue; } @@ -122,31 +129,55 @@ pub fn serve(project: &Path, storage: Option<&Path>) -> Result<(), String> { ("id".into(), id), ("result".into(), result), ]); - write_message(&mut output, &json::to_string(&resp).map_err(|e| e.message)?).map_err(|e| e.to_string())?; + write_message(&mut output, &json::to_string(&resp).map_err(|e| e.message)?, newline).map_err(|e| e.to_string())?; } } Ok(()) } pub fn read_message(input: &mut impl Read) -> io::Result> { - let mut header = Vec::new(); + Ok(read_incoming(input)?.map(|(body, _)| body)) +} + +fn read_incoming(input: &mut impl Read) -> io::Result> { + let mut buf = Vec::new(); let mut byte = [0u8; 1]; loop { let n = input.read(&mut byte)?; if n == 0 { - return Ok(None); + if buf.is_empty() { + return Ok(None); + } + break; + } + if buf.is_empty() && byte[0].is_ascii_whitespace() { + continue; } - header.push(byte[0]); - if header.ends_with(b"\r\n\r\n") { + buf.push(byte[0]); + if buf[0] == b'{' || buf[0] == b'[' { + if byte[0] == b'\n' { + if buf.last() == Some(&b'\n') { buf.pop(); } + if buf.last() == Some(&b'\r') { buf.pop(); } + return Ok(Some((String::from_utf8_lossy(&buf).into_owned(), true))); + } + if buf.len() > 8 * 1024 * 1024 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "body too large")); + } + continue; + } + if buf.ends_with(b"\r\n\r\n") || buf.ends_with(b"\n\n") { break; } - if header.len() > 64 * 1024 { + if buf.len() > 64 * 1024 { return Err(io::Error::new(io::ErrorKind::InvalidData, "header too large")); } } - let text = String::from_utf8_lossy(&header); + if buf.is_empty() { + return Ok(None); + } + let text = String::from_utf8_lossy(&buf); let mut len = 0usize; - for line in text.split("\r\n") { + for line in text.split(['\r', '\n']) { if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { len = v.trim().parse().unwrap_or(0); } @@ -156,8 +187,10 @@ pub fn read_message(input: &mut impl Read) -> io::Result> { return Err(io::Error::new(io::ErrorKind::InvalidData, "body too large")); } let mut body = vec![0u8; len]; - input.read_exact(&mut body)?; - Ok(Some(String::from_utf8_lossy(&body).into_owned())) + if len > 0 { + input.read_exact(&mut body)?; + } + Ok(Some((String::from_utf8_lossy(&body).into_owned(), false))) } fn discard_bytes(input: &mut impl Read, mut left: usize) -> io::Result<()> { @@ -173,7 +206,11 @@ fn discard_bytes(input: &mut impl Read, mut left: usize) -> io::Result<()> { Ok(()) } -pub fn write_message(output: &mut impl Write, body: &str) -> io::Result<()> { - write!(output, "Content-Length: {}\r\n\r\n{}", body.len(), body)?; +pub fn write_message(output: &mut impl Write, body: &str, newline: bool) -> io::Result<()> { + if newline { + writeln!(output, "{body}")?; + } else { + write!(output, "Content-Length: {}\r\n\r\n{}", body.len(), body)?; + } output.flush() } diff --git a/src/next_steps.rs b/src/next_steps.rs index 0934c9d..ab64b2d 100644 --- a/src/next_steps.rs +++ b/src/next_steps.rs @@ -201,3 +201,57 @@ fn hints_job_status(result: &Value) -> Vec { _ => Vec::new(), } } + +/// Warn when a JS/TS file builds code with eval or `new Function`, which static edges cannot see. +pub fn eval_warning(project: &std::path::Path, file_path: Option<&str>) -> Option { + let file_path = file_path?; + let lower = file_path.to_ascii_lowercase(); + if !lower.ends_with(".js") && !lower.ends_with(".jsx") && !lower.ends_with(".ts") && !lower.ends_with(".tsx") && !lower.ends_with(".mjs") && !lower.ends_with(".cjs") { + return None; + } + let text = std::fs::read_to_string(project.join(file_path)).ok()?; + if !has_eval_or_function(&text) { + return None; + } + Some(Value::object(vec![ + ("action".into(), Value::String("warn_eval_used".into())), + ("reason".into(), Value::String(format!( + "{file_path} uses eval(...) or new Function(...). Modules loaded that way are invisible to static analysis, so tests of runtime-generated targets are not in this list. Cross-check risk_map → unknown_require_count for affected files." + ))), + ])) +} + +fn has_eval_or_function(text: &str) -> bool { + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i..].starts_with(b"eval") && (i == 0 || !is_ident(bytes[i - 1])) { + let mut j = i + 4; + while j < bytes.len() && bytes[j].is_ascii_whitespace() { j += 1; } + if j < bytes.len() && bytes[j] == b'(' { + return true; + } + } + if bytes[i..].starts_with(b"new") && (i == 0 || !is_ident(bytes[i - 1])) { + let mut j = i + 3; + while j < bytes.len() && bytes[j].is_ascii_whitespace() { j += 1; } + if bytes.get(j..).is_some_and(|rest| rest.starts_with(b"Function")) { + let after = j + "Function".len(); + let boundary_ok = after == bytes.len() || !is_ident(bytes[after]); + if boundary_ok { + let mut k = after; + while k < bytes.len() && bytes[k].is_ascii_whitespace() { k += 1; } + if k < bytes.len() && bytes[k] == b'(' { + return true; + } + } + } + } + i += 1; + } + false +} + +fn is_ident(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b == b'$' +} diff --git a/src/run_cmd.rs b/src/run_cmd.rs new file mode 100644 index 0000000..e9372a8 --- /dev/null +++ b/src/run_cmd.rs @@ -0,0 +1,212 @@ +//! `chisel run`: stream the test process and record pytest and Jest results. + +use crate::cli::Args; +use crate::engine::Engine; +use crate::json; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::thread; + +pub fn run_tests(args: &Args) -> Result { + if args.test_command.is_empty() { + return Err("chisel run requires a command after --".into()); + } + let mut cmd_args = args.test_command.clone(); + let program = cmd_args.remove(0); + let framework = detect_framework(&program, &cmd_args); + if framework.is_empty() { + println!("Warning: could not detect test framework; results will not be recorded."); + println!("Running: {} {}", program, cmd_args.join(" ")); + } + augment(&mut cmd_args, framework); + let mut jest_file = None; + if framework == "jest" && !cmd_args.iter().any(|a| a == "--outputFile" || a == "-o") { + let path = std::env::temp_dir().join(format!("chisel-jest-{}-{}.json", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0))); + cmd_args.push("--outputFile".into()); + cmd_args.push(path.display().to_string()); + jest_file = Some(path); + } + let mut child = Command::new(&program) + .args(&cmd_args) + .current_dir(&args.project_dir) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| e.to_string())?; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let out_buf = Arc::new(Mutex::new(Vec::new())); + let err_buf = Arc::new(Mutex::new(Vec::new())); + let out_copy = Arc::clone(&out_buf); + let err_copy = Arc::clone(&err_buf); + let out_thread = thread::spawn(move || { + if let Some(mut pipe) = stdout { + stream_pipe(&mut pipe, &mut io::stdout(), &out_copy); + } + }); + let err_thread = thread::spawn(move || { + if let Some(mut pipe) = stderr { + stream_pipe(&mut pipe, &mut io::stdout(), &err_copy); + } + }); + let status = child.wait().map_err(|e| e.to_string())?; + let _ = out_thread.join(); + let _ = err_thread.join(); + let code = status.code().unwrap_or(1); + if framework == "pytest" || framework == "jest" { + let stdout_text = String::from_utf8_lossy(&out_buf.lock().unwrap_or_else(|e| e.into_inner())).into_owned(); + let stderr_text = String::from_utf8_lossy(&err_buf.lock().unwrap_or_else(|e| e.into_inner())).into_owned(); + let combined = format!("{stdout_text}{stderr_text}"); + let rows = if framework == "jest" { + let from_file = jest_file.as_ref().and_then(|p| std::fs::read_to_string(p).ok()).unwrap_or_default(); + let text = if from_file.is_empty() { combined } else { from_file }; + parse_jest_json(&text, Some(&args.project_dir)) + } else { + combined.lines().filter_map(parse_pytest_line).collect() + }; + let mut engine = Engine::open(&args.project_dir, args.storage_dir.as_deref())?; + if !rows.is_empty() { + engine.record_results_batch(&rows)?; + println!("\n[chisel] Recorded {} test result(s)", rows.len()); + } else { + println!("\n[chisel] No test results parsed from output"); + } + } + if let Some(path) = jest_file { + let _ = std::fs::remove_file(path); + } + Ok(code) +} + +fn stream_pipe(pipe: &mut impl Read, out: &mut impl Write, buf: &Mutex>) { + let mut tmp = [0u8; 4096]; + loop { + let n = match pipe.read(&mut tmp) { + Ok(0) => break, + Ok(n) => n, + Err(_) => break, + }; + let _ = out.write_all(&tmp[..n]); + let _ = out.flush(); + if let Ok(mut guard) = buf.lock() { + guard.extend_from_slice(&tmp[..n]); + } + } +} + +fn detect_framework(program: &str, args: &[String]) -> &'static str { + let base = Path::new(program).file_name().and_then(|s| s.to_str()).unwrap_or(program).to_ascii_lowercase(); + if base == "pytest" { + "pytest" + } else if base == "jest" || base == "npx" { + "jest" + } else if base == "go" { + "go" + } else if base == "cargo" { + "cargo" + } else if args.iter().any(|a| a == "jest" || a.rsplit(['/', '\\']).next() == Some("jest")) { + "jest" + } else { + "" + } +} + +fn augment(args: &mut Vec, framework: &str) { + match framework { + "pytest" => { + if !args.iter().any(|a| a == "-v" || a == "--verbose") { + args.push("-v".into()); + } + } + "jest" => { + if !args.iter().any(|a| a == "--json") { + args.push("--json".into()); + } + } + "go" => { + if !args.iter().any(|a| a == "-json") { + args.push("-json".into()); + } + } + "cargo" if !args.iter().any(|a| a == "--message-format=json" || a.starts_with("--message-format")) => { + args.push("--message-format=json".into()); + } + _ => {} + } +} + +pub fn parse_jest_json(text: &str, project: Option<&Path>) -> Vec<(String, bool)> { + let Ok(doc) = json::parse(text) else { return Vec::new() }; + let Some(suites) = doc.get("testResults").and_then(|v| v.as_array()) else { return Vec::new() }; + let mut out = Vec::new(); + for suite in suites { + let name = suite.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let name = relativize(name, project); + let Some(assertions) = suite.get("assertionResults").and_then(|v| v.as_array()) else { continue }; + for assertion in assertions { + let title = assertion.get("title").and_then(|v| v.as_str()).unwrap_or(""); + let passed = assertion.get("status").and_then(|v| v.as_str()) == Some("passed"); + out.push((format!("{name}:{title}"), passed)); + } + } + out +} + +fn relativize(name: &str, project: Option<&Path>) -> String { + let name = name.replace('\\', "/"); + let Some(project) = project else { return name }; + let path = PathBuf::from(&name); + if let Ok(rel) = path.strip_prefix(project) { + return rel.to_string_lossy().replace('\\', "/"); + } + if let Ok(root) = project.canonicalize() { + if let Ok(rel) = PathBuf::from(&name).strip_prefix(&root) { + return rel.to_string_lossy().replace('\\', "/"); + } + } + name +} + +pub fn parse_pytest_line(line: &str) -> Option<(String, bool)> { + let line = line.trim(); + if line.is_empty() { + return None; + } + let raw: Vec<&str> = line.split_whitespace().collect(); + let tokens = trim_progress(&raw); + if tokens.len() < 2 { + return None; + } + let (idx, status) = if is_status(tokens[tokens.len() - 1]) { + (tokens.len() - 1, tokens[tokens.len() - 1]) + } else { + return None; + }; + let id = tokens[..idx].join(" "); + if id.is_empty() || id.chars().all(|c| c == '_' || c == '=' || c == '-') { + return None; + } + if !id.contains(':') && !id.contains('/') && !id.contains('.') { + return None; + } + let passed = matches!(status, "PASSED" | "XFAIL"); + Some((id, passed)) +} + +fn is_status(token: &str) -> bool { + matches!(token, "PASSED" | "FAILED" | "ERROR" | "SKIPPED" | "XFAIL" | "XPASS") +} + +fn trim_progress<'a>(tokens: &'a [&'a str]) -> &'a [&'a str] { + if tokens.len() >= 2 { + let last = tokens[tokens.len() - 1]; + let prev = tokens[tokens.len() - 2]; + if (last.ends_with(']') && prev.starts_with('[')) || (last.starts_with('[') && last.ends_with(']')) { + let end = if last.starts_with('[') && last.ends_with(']') { tokens.len() - 1 } else { tokens.len() - 2 }; + return &tokens[..end]; + } + } + tokens +} diff --git a/src/sqlite_import.rs b/src/sqlite_import.rs new file mode 100644 index 0000000..082dca5 --- /dev/null +++ b/src/sqlite_import.rs @@ -0,0 +1,468 @@ +//! One-shot reader for recorded results in a legacy SQLite `chisel.db`. +//! The live snapshot stays `.chisel/store`. This module never writes SQLite. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct LegacyResult { + pub test_id: String, + pub passed: bool, + pub duration_ms: Option, + pub recorded_at: f64, +} + +pub fn read_test_results(path: &Path) -> Result, String> { + let mut bytes = fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?; + if bytes.len() < 100 || &bytes[..16] != b"SQLite format 3\0" { + return Err("not a SQLite database".into()); + } + apply_wal(path, &mut bytes); + let page_size = page_size_of(&bytes)?; + let reserved = bytes[20] as usize; + let master = table_rows(&bytes, page_size, reserved, 1)?; + let mut root = None; + let mut columns = Vec::new(); + for row in master { + let typ = row.first().and_then(Cell::as_text).unwrap_or(""); + let name = row.get(1).and_then(Cell::as_text).unwrap_or(""); + if typ == "table" && name == "test_results" { + root = row.get(3).and_then(Cell::as_int); + if let Some(sql) = row.get(4).and_then(Cell::as_text) { + columns = column_names(sql); + } + break; + } + } + let root = root.ok_or_else(|| "test_results table not found".to_string())?; + if root < 1 { + return Err("test_results has no root page".into()); + } + let idx = |name: &str, fallback: usize| -> usize { + columns.iter().position(|c| c == name).unwrap_or(fallback) + }; + let id_i = idx("test_id", 0); + let passed_i = idx("passed", 1); + let dur_i = idx("duration_ms", 2); + let at_i = idx("recorded_at", 3); + let mut out = Vec::new(); + for row in table_rows(&bytes, page_size, reserved, root as usize)? { + let Some(test_id) = row.get(id_i).and_then(Cell::as_text) else { continue }; + if test_id.is_empty() { + continue; + } + let passed = row.get(passed_i).and_then(Cell::as_int).unwrap_or(0) != 0; + let duration_ms = row.get(dur_i).and_then(Cell::as_number); + let recorded_at = match row.get(at_i) { + Some(Cell::Text(s)) => parse_recorded_at(s).unwrap_or(0.0), + Some(Cell::Int(n)) => *n as f64, + Some(Cell::Float(n)) => *n, + _ => 0.0, + }; + out.push(LegacyResult { + test_id: test_id.to_string(), + passed, + duration_ms, + recorded_at, + }); + } + out.sort_by(|a, b| a.recorded_at.partial_cmp(&b.recorded_at).unwrap_or(std::cmp::Ordering::Equal).then(a.test_id.cmp(&b.test_id))); + Ok(out) +} + +fn page_size_of(bytes: &[u8]) -> Result { + let raw = u16::from_be_bytes([bytes[16], bytes[17]]); + let size = if raw == 1 { 65536 } else { raw as usize }; + if size < 512 || !size.is_power_of_two() { + return Err(format!("bad page size {size}")); + } + Ok(size) +} + +#[derive(Clone, Debug)] +enum Cell { + Null, + Int(i64), + Float(f64), + Text(String), + Blob, +} + +impl Cell { + fn as_text(&self) -> Option<&str> { + match self { + Cell::Text(s) => Some(s), + _ => None, + } + } + fn as_int(&self) -> Option { + match self { + Cell::Int(n) => Some(*n), + Cell::Float(n) => Some(*n as i64), + _ => None, + } + } + fn as_number(&self) -> Option { + match self { + Cell::Int(n) => Some(*n as f64), + Cell::Float(n) => Some(*n), + _ => None, + } + } +} + +fn table_rows(bytes: &[u8], page_size: usize, reserved: usize, root: usize) -> Result>, String> { + let mut rows = Vec::new(); + let mut stack = vec![root]; + let mut guard = 0; + while let Some(page_no) = stack.pop() { + guard += 1; + if guard > 100_000 { + return Err("sqlite page walk too deep".into()); + } + let page = page_bytes(bytes, page_size, page_no)?; + let header_at = if page_no == 1 { 100 } else { 0 }; + if header_at >= page.len() { + continue; + } + let kind = page[header_at]; + let cell_count = u16::from_be_bytes([page[header_at + 3], page[header_at + 4]]) as usize; + let interior = kind == 0x05; + let ptr_at = header_at + if interior { 12 } else { 8 }; + if interior { + let right = u32::from_be_bytes([page[header_at + 8], page[header_at + 9], page[header_at + 10], page[header_at + 11]]) as usize; + if right > 0 { + stack.push(right); + } + } + if kind != 0x0d && kind != 0x05 { + continue; + } + for n in 0..cell_count { + let at = ptr_at + n * 2; + if at + 2 > page.len() { + break; + } + let cell_off = u16::from_be_bytes([page[at], page[at + 1]]) as usize; + if interior { + if let Some(child) = interior_child(page, cell_off) { + stack.push(child); + } + } else if let Some(payload) = leaf_payload(bytes, page, page_size, reserved, cell_off) { + rows.push(decode_record(&payload)); + } + } + } + Ok(rows) +} + +fn page_bytes(bytes: &[u8], page_size: usize, page_no: usize) -> Result<&[u8], String> { + if page_no == 0 { + return Err("sqlite page 0".into()); + } + let start = (page_no - 1) * page_size; + let end = start + page_size; + if end > bytes.len() { + return Err(format!("sqlite page {page_no} is past the end of the file")); + } + Ok(&bytes[start..end]) +} + +fn interior_child(page: &[u8], off: usize) -> Option { + if off + 4 > page.len() { + return None; + } + Some(u32::from_be_bytes([page[off], page[off + 1], page[off + 2], page[off + 3]]) as usize) +} + +fn leaf_payload(file: &[u8], page: &[u8], page_size: usize, reserved: usize, off: usize) -> Option> { + let (payload_size, next) = read_varint(page, off)?; + let (_rowid, next) = read_varint(page, next)?; + let payload_size = payload_size as usize; + let local = local_payload(page_size, reserved, payload_size); + if next + local > page.len() { + return None; + } + let mut payload = page[next..next + local].to_vec(); + if payload.len() < payload_size { + if next + local + 4 > page.len() { + return None; + } + let mut overflow = u32::from_be_bytes([ + page[next + local], + page[next + local + 1], + page[next + local + 2], + page[next + local + 3], + ]) as usize; + let usable = page_size.saturating_sub(reserved); + while payload.len() < payload_size && overflow > 0 { + let ov = page_bytes(file, page_size, overflow).ok()?; + if ov.len() < 4 { + break; + } + let take = (payload_size - payload.len()).min(usable.saturating_sub(4)); + payload.extend_from_slice(&ov[4..4 + take]); + overflow = u32::from_be_bytes([ov[0], ov[1], ov[2], ov[3]]) as usize; + } + } + payload.truncate(payload_size); + Some(payload) +} + +fn local_payload(page_size: usize, reserved: usize, payload: usize) -> usize { + let usable = page_size.saturating_sub(reserved).max(1); + let max_local = usable.saturating_sub(35); + if payload <= max_local { + return payload; + } + let min_local = ((usable.saturating_sub(12)) * 32 / 255).saturating_sub(23); + let mut local = min_local + ((payload - min_local) % usable.saturating_sub(4).max(1)); + if local > max_local { + local = min_local; + } + local.min(payload) +} + +fn decode_record(payload: &[u8]) -> Vec { + let Some((header_size, header_start)) = read_varint(payload, 0) else { + return Vec::new(); + }; + let header_size = header_size as usize; + if header_size > payload.len() || header_start > header_size { + return Vec::new(); + } + let mut types = Vec::new(); + let mut i = header_start; + while i < header_size { + let Some((serial, next)) = read_varint(payload, i) else { break }; + types.push(serial); + i = next; + } + let mut body = header_size; + let mut cells = Vec::new(); + for serial in types { + let (cell, used) = read_serial(payload, body, serial); + cells.push(cell); + body += used; + } + cells +} + +fn read_serial(payload: &[u8], at: usize, serial: u64) -> (Cell, usize) { + let take = |n: usize| -> Option<&[u8]> { + payload.get(at..at + n) + }; + match serial { + 0 => (Cell::Null, 0), + 1 => (Cell::Int(take(1).map(|b| b[0] as i8 as i64).unwrap_or(0)), 1), + 2 => (Cell::Int(take(2).map(|b| i16::from_be_bytes([b[0], b[1]]) as i64).unwrap_or(0)), 2), + 3 => { + let b = take(3).unwrap_or(&[0, 0, 0]); + let mut raw = [0u8; 4]; + raw[1..].copy_from_slice(&b[..3.min(b.len())]); + let mut n = i32::from_be_bytes(raw) >> 8; + if b.len() == 3 && b[0] & 0x80 != 0 { + n |= !0 << 24; + } + (Cell::Int(n as i64), 3) + } + 4 => (Cell::Int(take(4).map(|b| i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64).unwrap_or(0)), 4), + 5 => { + let b = take(6).unwrap_or(&[0; 6]); + let mut raw = [0u8; 8]; + if b.len() == 6 { + raw[2..].copy_from_slice(b); + } + let mut n = i64::from_be_bytes(raw) >> 16; + if b.len() == 6 && b[0] & 0x80 != 0 { + n |= !0 << 48; + } + (Cell::Int(n), 6) + } + 6 => (Cell::Int(take(8).map(|b| i64::from_be_bytes(b.try_into().unwrap_or([0; 8]))).unwrap_or(0)), 8), + 7 => (Cell::Float(take(8).map(|b| f64::from_be_bytes(b.try_into().unwrap_or([0; 8]))).unwrap_or(0.0)), 8), + 8 => (Cell::Int(0), 0), + 9 => (Cell::Int(1), 0), + n if n >= 12 && n % 2 == 0 => { + let len = ((n - 12) / 2) as usize; + (Cell::Blob, len) + } + n if n >= 13 && n % 2 == 1 => { + let len = ((n - 13) / 2) as usize; + let text = payload.get(at..at + len).map(|b| String::from_utf8_lossy(b).into_owned()).unwrap_or_default(); + (Cell::Text(text), len) + } + _ => (Cell::Null, 0), + } +} + +fn read_varint(data: &[u8], mut i: usize) -> Option<(u64, usize)> { + let mut value = 0u64; + for n in 0..9 { + let b = *data.get(i)?; + i += 1; + if n == 8 { + value = (value << 8) | b as u64; + return Some((value, i)); + } + value = (value << 7) | (b & 0x7f) as u64; + if b & 0x80 == 0 { + return Some((value, i)); + } + } + None +} + +fn column_names(sql: &str) -> Vec { + let Some(start) = sql.find('(') else { return Vec::new() }; + let body = &sql[start + 1..]; + let Some(end) = body.rfind(')') else { return Vec::new() }; + let mut names = Vec::new(); + for part in body[..end].split(',') { + let part = part.trim(); + if part.is_empty() || part.to_ascii_lowercase().starts_with("primary ") || part.to_ascii_lowercase().starts_with("unique ") || part.to_ascii_lowercase().starts_with("constraint ") { + continue; + } + let name = part.split_whitespace().next().unwrap_or("").trim_matches(|c| c == '"' || c == '\'' || c == '`'); + if !name.is_empty() { + names.push(name.to_string()); + } + } + names +} + +pub fn parse_recorded_at(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() { + return None; + } + if let Ok(n) = text.parse::() { + if n > 1_000_000_000.0 { + return Some(n); + } + } + let (date, rest) = text.split_once('T').unwrap_or((text, "00:00:00")); + let mut date_parts = date.split('-'); + let year: i32 = date_parts.next()?.parse().ok()?; + let month: u32 = date_parts.next()?.parse().ok()?; + let day: u32 = date_parts.next()?.parse().ok()?; + let rest = rest.trim_end_matches('Z'); + let (time, offset) = if let Some(i) = rest.rfind('+') { + (&rest[..i], Some(&rest[i..])) + } else if let Some(i) = rest.rfind('-') { + if i > 0 { (&rest[..i], Some(&rest[i..])) } else { (rest, None) } + } else { + (rest, None) + }; + let mut time_parts = time.split(':'); + let hour: u32 = time_parts.next()?.parse().ok()?; + let min: u32 = time_parts.next().unwrap_or("0").parse().ok()?; + let sec_part = time_parts.next().unwrap_or("0"); + let (sec_txt, frac_txt) = sec_part.split_once('.').unwrap_or((sec_part, "0")); + let sec: u32 = sec_txt.parse().ok()?; + let frac: f64 = format!("0.{frac_txt}").parse().unwrap_or(0.0); + let mut unix = civil_unix(year, month, day, hour, min, sec)? as f64 + frac; + if let Some(off) = offset { + let sign = if off.starts_with('-') { -1.0 } else { 1.0 }; + let off = off.trim_start_matches(['+', '-']); + let mut p = off.split(':'); + let oh: f64 = p.next()?.parse().ok()?; + let om: f64 = p.next().unwrap_or("0").parse().unwrap_or(0.0); + unix -= sign * (oh * 3600.0 + om * 60.0); + } + Some(unix) +} + +fn civil_unix(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> Option { + if !(1..=12).contains(&month) || day == 0 || day > 31 || hour > 23 || min > 59 || sec > 60 { + return None; + } + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = (y - era * 400) as u32; + let mp = if month > 2 { month - 3 } else { month + 9 }; + let doy = (153 * mp + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era as i64 * 146097 + doe as i64 - 719468; + Some(days * 86400 + hour as i64 * 3600 + min as i64 * 60 + sec as i64) +} + +fn apply_wal(db_path: &Path, bytes: &mut Vec) { + let wal_path = wal_sibling(db_path); + let Ok(wal) = fs::read(&wal_path) else { return }; + if wal.len() < 32 { + return; + } + let magic = u32::from_be_bytes([wal[0], wal[1], wal[2], wal[3]]); + if magic != 0x377f_0682 && magic != 0x377f_0683 { + return; + } + let page_size = u32::from_be_bytes([wal[8], wal[9], wal[10], wal[11]]) as usize; + if page_size < 512 || wal.len() < 32 + 24 + page_size { + return; + } + let salt1 = u32::from_be_bytes([wal[16], wal[17], wal[18], wal[19]]); + let salt2 = u32::from_be_bytes([wal[20], wal[21], wal[22], wal[23]]); + let frame = 24 + page_size; + let mut latest: BTreeMap> = BTreeMap::new(); + let mut committed: BTreeMap> = BTreeMap::new(); + let mut off = 32; + while off + frame <= wal.len() { + let page_no = u32::from_be_bytes([wal[off], wal[off + 1], wal[off + 2], wal[off + 3]]) as usize; + let db_size = u32::from_be_bytes([wal[off + 4], wal[off + 5], wal[off + 6], wal[off + 7]]); + let s1 = u32::from_be_bytes([wal[off + 8], wal[off + 9], wal[off + 10], wal[off + 11]]); + let s2 = u32::from_be_bytes([wal[off + 12], wal[off + 13], wal[off + 14], wal[off + 15]]); + if s1 != salt1 || s2 != salt2 || page_no == 0 { + break; + } + let page = wal[off + 24..off + 24 + page_size].to_vec(); + latest.insert(page_no, page); + if db_size > 0 { + let need = db_size as usize * page_size; + if bytes.len() < need { + bytes.resize(need, 0); + } + committed.clone_from(&latest); + } + off += frame; + } + for (page_no, page) in committed { + let start = (page_no - 1) * page_size; + let end = start + page_size; + if end <= bytes.len() && page.len() == page_size { + bytes[start..end].copy_from_slice(&page); + } + } +} + +fn wal_sibling(path: &Path) -> std::path::PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push("-wal"); + std::path::PathBuf::from(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_legacy_fixture() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/legacy/chisel.sqlite"); + let rows = read_test_results(&path).unwrap(); + assert_eq!(rows.len(), 3, "{rows:?}"); + assert_eq!(rows[0].test_id, "tests/test_calc.py::test_add"); + assert!(rows[0].passed); + assert_eq!(rows[0].duration_ms, Some(12.0)); + assert!(rows[0].recorded_at > 1_700_000_000.0); + let sub = rows.iter().find(|r| r.test_id.ends_with("test_sub")).unwrap(); + assert!(!sub.passed); + assert!(sub.duration_ms.is_none()); + } + + #[test] + fn parses_iso_timestamps() { + let unix = parse_recorded_at("2026-07-01T00:00:00+00:00").unwrap(); + assert!((unix - 1_782_864_000.0).abs() < 1.0, "{unix}"); + } +} diff --git a/tests/fixtures/legacy/chisel.sqlite b/tests/fixtures/legacy/chisel.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..b21650712982379c1c6f35cccaac1fa81394aa20 GIT binary patch literal 12288 zcmeI$&r9P#6bJChMC%Wf*_+pQDO9YQwxAdfT4fN|+SQtc^%Pd*-8j{wd%oL}bx)F4-)Tr1J)zPp^*@LddGe1}rprLWOsQr#p-qt1nYt_P=OO2T$0QK9lp7oO#t-#cpUAJ$u^bRbWulvRt=hi@(} zuJgjeLD6E-g$i%=Rcv~BKj@~NEcQp=(65X}y}o|B=Q(etEq*fbDjv;(0Rad=00Izz z00bZa0SG_<0uX>euE3@}DQB#$UYD&`vtGL$#O;;KJJ-#kn; PathBuf { + let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let p = std::env::temp_dir().join(format!("chisel-parity-{name}-{n}")); + fs::create_dir_all(&p).unwrap(); + p +} + +fn git(dir: &Path, args: &[&str]) { + let status = Command::new("git").args(args).current_dir(dir).status().unwrap(); + assert!(status.success(), "git {args:?}"); +} + +fn init_repo(dir: &Path) { + git(dir, &["init", "-b", "main"]); + git(dir, &["config", "user.email", "a@example.com"]); + git(dir, &["config", "user.name", "Ada"]); + git(dir, &["config", "commit.gpgsign", "false"]); + git(dir, &["config", "core.autocrlf", "false"]); +} + +fn commit_all(dir: &Path, msg: &str) { + git(dir, &["add", "-A"]); + let status = Command::new("git") + .args(["commit", "-m", msg]) + .current_dir(dir) + .env("GIT_AUTHOR_DATE", "2024-05-01T00:00:00+00:00") + .env("GIT_COMMITTER_DATE", "2024-05-01T00:00:00+00:00") + .status() + .unwrap(); + assert!(status.success(), "{msg:?}"); +} + +fn write_sample(dir: &Path) { + fs::create_dir_all(dir.join("py")).unwrap(); + fs::create_dir_all(dir.join("js")).unwrap(); + fs::create_dir_all(dir.join("java")).unwrap(); + fs::create_dir_all(dir.join("cs")).unwrap(); + fs::create_dir_all(dir.join("swift")).unwrap(); + fs::create_dir_all(dir.join("cpp")).unwrap(); + fs::create_dir_all(dir.join("rb")).unwrap(); + fs::create_dir_all(dir.join("tests")).unwrap(); + fs::write(dir.join("py/a.py"), "from py.b import pong\n\ndef ping():\n return pong()\n").unwrap(); + fs::write(dir.join("py/b.py"), "from py.a import ping\n\ndef pong():\n return ping()\n").unwrap(); + fs::write(dir.join("py/util.py"), "def util():\n return 3\n").unwrap(); + fs::write(dir.join("tests/test_a.py"), "from py.a import ping\n\ndef test_ping():\n assert ping() == 1\n").unwrap(); + fs::write(dir.join("js/math.js"), "function multiplies(a, b) {\n return a * b;\n}\nmodule.exports = { multiplies };\n").unwrap(); + fs::write(dir.join("js/math.test.js"), "const math = require('./math');\ntest('multiplies', () => { math.multiplies(2, 3); });\n").unwrap(); + fs::write(dir.join("js/flow.spec.ts"), "import { test } from '@playwright/test';\ntest('loads', async () => {});\n").unwrap(); + fs::write(dir.join("java/CalcTest.java"), "import org.junit.Test;\npublic class CalcTest {\n @Test\n public void shouldAdd() {}\n}\n").unwrap(); + fs::write(dir.join("cs/CalcTests.cs"), "public class CalcTests {\n [Fact]\n public void AddsNumbers() {}\n}\n").unwrap(); + fs::write(dir.join("swift/CalcTests.swift"), "import Testing\n@Test func addsNumbers() {}\n").unwrap(); + fs::write(dir.join("cpp/calc_unittest.cpp"), "#include \nTEST(Calc, Adds) {\n EXPECT_EQ(1, 1);\n}\n").unwrap(); + fs::write(dir.join("rb/calc_spec.rb"), "describe 'calc' do\n it 'adds' do\n end\nend\n").unwrap(); + fs::write(dir.join("js/dynamic.js"), "function make() {\n return eval('1');\n}\n").unwrap(); +} + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_chisel") +} + +#[test] +fn newline_mcp_answers_initialize() { + let dir = scratch("mcp"); + let store = dir.join("store"); + let mut child = Command::new(bin()) + .args(["serve-mcp"]) + .env("CHISEL_PROJECT_DIR", &dir) + .env("CHISEL_STORAGE_DIR", &store) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let init = b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}\n"; + child.stdin.as_mut().unwrap().write_all(init).unwrap(); + drop(child.stdin.take()); + let mut out = String::new(); + child.stdout.take().unwrap().read_to_string(&mut out).unwrap(); + let _ = child.wait(); + assert!(out.contains("protocolVersion"), "{out:?}"); + assert!(out.contains(env!("CARGO_PKG_VERSION")), "{out:?}"); + assert!(!out.contains("Content-Length"), "newline clients must not get Content-Length framing: {out:?}"); + let tools = b"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\"}\n"; + let framed = { + let mut input = std::io::Cursor::new(tools.to_vec()); + chisel::mcp::read_message(&mut input).unwrap().unwrap() + }; + assert!(framed.contains("tools/list"), "{framed:?}"); +} + +#[test] +fn content_length_round_trip_still_parses() { + let body = "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"ping\"}"; + let mut raw = format!("Content-Length: {}\r\n\r\n{body}", body.len()).into_bytes(); + raw.extend_from_slice(b"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"ping\"}\n"); + let mut input = std::io::Cursor::new(raw); + assert_eq!(chisel::mcp::read_message(&mut input).unwrap().unwrap(), body); + assert!(chisel::mcp::read_message(&mut input).unwrap().unwrap().contains("\"id\":2")); +} + +#[test] +fn flags_reach_the_engine_on_the_sample() { + let dir = scratch("flags"); + write_sample(&dir); + init_repo(&dir); + commit_all(&dir, "init"); + let calc = dir.join("py/a.py"); + let mut text = fs::read_to_string(&calc).unwrap(); + text.push_str("\n# again\n"); + fs::write(&calc, text).unwrap(); + git(&dir, &["add", "-A"]); + let status = Command::new("git") + .args(["commit", "-m", "edit ping"]) + .current_dir(&dir) + .env("GIT_AUTHOR_DATE", "2024-06-01T00:00:00+00:00") + .env("GIT_COMMITTER_DATE", "2024-06-01T00:00:00+00:00") + .status() + .unwrap(); + assert!(status.success()); + let store = dir.join("store"); + let analyze = Command::new(bin()).args(["--json", "analyze", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(analyze.status.success(), "{}", String::from_utf8_lossy(&analyze.stderr)); + let snap = fs::read_to_string(store.join("store")).unwrap(); + assert!(snap.contains("shouldAdd"), "{snap:?}"); + assert!(snap.contains("AddsNumbers"), "{snap:?}"); + assert!(snap.contains("addsNumbers"), "{snap:?}"); + assert!(snap.contains("Calc.Adds"), "{snap:?}"); + assert!(snap.contains("\"framework\":\"junit\""), "{snap:?}"); + assert!(snap.contains("\"framework\":\"csharp_test\""), "{snap:?}"); + assert!(snap.contains("\"framework\":\"gtest\""), "{snap:?}"); + assert!(snap.contains("\"framework\":\"jest\""), "{snap:?}"); + assert!(snap.contains("\"framework\":\"playwright\""), "{snap:?}"); + assert!(snap.contains("\"framework\":\"rspec\""), "{snap:?}"); + assert!(!snap.contains("require:function"), "require() must not become a code unit: {snap:?}"); + + let risk_all = Command::new(bin()).args(["--json", "risk-map", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let risk_text = String::from_utf8_lossy(&risk_all.stdout); + let risk = chisel::json::parse(risk_text.trim()).unwrap(); + let files = risk.get("files").and_then(|v| v.as_array()).unwrap(); + let paths: Vec<&str> = files.iter().filter_map(|f| f.get("file_path").and_then(|v| v.as_str())).collect(); + assert!(paths.iter().any(|p| p.ends_with("a.py")) && paths.iter().any(|p| p.ends_with("b.py")), "{paths:?}"); + assert!(!paths.iter().any(|p| p.contains("calc_unittest")), "gtest file ranked as code: {paths:?}"); + let cycles = risk.get("_meta").and_then(|m| m.get("cycles")).and_then(|v| v.as_array()).unwrap(); + assert!(cycles.iter().any(|c| c.get("length").and_then(|v| v.as_i64()) == Some(2)), "{cycles:?}"); + + let scoped = Command::new(bin()).args(["--json", "risk-map", "py", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let scoped_doc = chisel::json::parse(String::from_utf8_lossy(&scoped.stdout).trim()).unwrap(); + let scoped_files = scoped_doc.get("files").and_then(|v| v.as_array()).unwrap(); + assert!(scoped_files.iter().all(|f| f.get("file_path").and_then(|v| v.as_str()).unwrap_or("").starts_with("py/")), "{scoped_doc:?}"); + assert!(scoped_files.len() < files.len(), "directory scope did not shrink the map"); + + let with_tests = Command::new(bin()).args(["--json", "risk-map", "--no-exclude-tests", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(with_tests.status.success(), "{}", String::from_utf8_lossy(&with_tests.stderr)); + let with_doc = chisel::json::parse(String::from_utf8_lossy(&with_tests.stdout).trim()).unwrap(); + let with_paths: Vec<&str> = with_doc.get("files").and_then(|v| v.as_array()).unwrap().iter().filter_map(|f| f.get("file_path").and_then(|v| v.as_str())).collect(); + assert!(with_paths.iter().any(|p| p.contains("test_a.py")), "{with_paths:?}"); + + let gaps = Command::new(bin()).args(["--json", "test-gaps", "py/util.py", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let gap_doc = chisel::json::parse(String::from_utf8_lossy(&gaps.stdout).trim()).unwrap(); + let gap_rows = gap_doc.as_array().unwrap(); + assert!(!gap_rows.is_empty(), "{gap_doc:?}"); + assert!(gap_rows.iter().all(|g| g.get("file_path").and_then(|v| v.as_str()) == Some("py/util.py")), "{gap_doc:?}"); + + let triage = Command::new(bin()).args(["--json", "triage", "py", "--top-n", "1", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(triage.status.success(), "{}", String::from_utf8_lossy(&triage.stderr)); + let triage_doc = chisel::json::parse(String::from_utf8_lossy(&triage.stdout).trim()).unwrap(); + assert_eq!(triage_doc.get("top_risk_files").and_then(|v| v.as_array()).unwrap().len(), 1); + + let suggest = Command::new(bin()).args(["--json", "suggest-tests", "--directory", "py", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(suggest.status.success(), "{}", String::from_utf8_lossy(&suggest.stderr)); + let suggest_doc = chisel::json::parse(String::from_utf8_lossy(&suggest.stdout).trim()).unwrap(); + assert!(suggest_doc.as_object().is_some(), "directory suggest must be a map: {suggest_doc:?}"); + + let fallback = Command::new(bin()).args(["--json", "suggest-tests", "py/util.py", "--fallback", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(fallback.status.success(), "{}", String::from_utf8_lossy(&fallback.stderr)); + + let impact = Command::new(bin()).args(["--json", "impact", "py/a.py", "--functions", "ping", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(impact.status.success(), "{}", String::from_utf8_lossy(&impact.stderr)); + assert!(String::from_utf8_lossy(&impact.stdout).contains("ping") || String::from_utf8_lossy(&impact.stdout).contains("[]"), "{}", String::from_utf8_lossy(&impact.stdout)); + + let churn = Command::new(bin()).args(["--json", "churn", "py/a.py", "--unit", "ping", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(churn.status.success(), "{}", String::from_utf8_lossy(&churn.stderr)); + let churn_doc = chisel::json::parse(String::from_utf8_lossy(&churn.stdout).trim()).unwrap(); + let churn_rows = churn_doc.as_array().unwrap(); + assert_eq!(churn_rows.len(), 1, "{churn_doc:?}"); + assert_eq!(churn_rows[0].get("unit_name").and_then(|v| v.as_str()), Some("ping")); + assert!(churn_rows[0].get("total_insertions").and_then(|v| v.as_i64()).unwrap_or(0) > 0, "{churn_doc:?}"); + assert!(churn_rows[0].get("last_changed").and_then(|v| v.as_str()).is_some(), "{churn_doc:?}"); + + let file_churn = Command::new(bin()).args(["--json", "churn", "py/a.py", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let file_rows = chisel::json::parse(String::from_utf8_lossy(&file_churn.stdout).trim()).unwrap(); + assert_eq!(file_rows.as_array().unwrap().len(), 1, "file churn should be the file-level row: {file_rows:?}"); + + let coupling = Command::new(bin()).args(["--json", "coupling", "py/a.py", "--min-count", "1", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(coupling.status.success(), "{}", String::from_utf8_lossy(&coupling.stderr)); + + fs::write(dir.join("py/new_mod.py"), "import a\n\ndef fresh():\n return a.ping()\n").unwrap(); + let disk = Command::new(bin()).args(["--json", "coupling", "py/new_mod.py", "--working-tree", "--min-count", "1", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(disk.status.success(), "{}", String::from_utf8_lossy(&disk.stderr)); + let disk_doc = chisel::json::parse(String::from_utf8_lossy(&disk.stdout).trim()).unwrap(); + assert!(disk_doc.get("note").and_then(|v| v.as_str()).unwrap_or("").contains("working_tree"), "{disk_doc:?}"); + assert!(!disk_doc.get("import_partners").and_then(|v| v.as_array()).unwrap().is_empty(), "{disk_doc:?}"); + + let recorded = Command::new(bin()).args(["--json", "record-result", "tests/test_a.py:test_ping", "--passed", "--duration", "15", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(recorded.status.success(), "{}", String::from_utf8_lossy(&recorded.stderr)); + let after = fs::read_to_string(store.join("store")).unwrap(); + assert!(after.contains("\"duration_ms\":15") || after.contains("\"duration_ms\": 15"), "{after:?}"); + + let job = Command::new(bin()).args(["--json", "start-job", "update", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(job.status.success(), "{}", String::from_utf8_lossy(&job.stderr)); + assert!(String::from_utf8_lossy(&job.stdout).contains("queued"), "{}", String::from_utf8_lossy(&job.stdout)); + + let stats = Command::new(bin()).args(["--json", "stats", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let stats_doc = chisel::json::parse(String::from_utf8_lossy(&stats.stdout).trim()).unwrap(); + assert!(matches!(stats_doc.get("co_change_query_min"), Some(chisel::json::Value::Int(_))), "{stats_doc:?}"); + assert!(stats_doc.get("shadow_graph").is_some(), "{stats_doc:?}"); + assert!(stats_doc.get("commit_files").is_some(), "{stats_doc:?}"); + assert!(stats_doc.get("churn_stats").is_some(), "{stats_doc:?}"); + + let update = Command::new(bin()).args(["--json", "update", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let update_doc = chisel::json::parse(String::from_utf8_lossy(&update.stdout).trim()).unwrap(); + if update_doc.get("files_updated").and_then(|v| v.as_i64()) == Some(0) { + assert_eq!(update_doc.get("code_units_found").and_then(|v| v.as_i64()), Some(0), "{update_doc:?}"); + } + assert!(update_doc.get("orphaned_results_cleaned").is_some(), "{update_doc:?}"); + assert!(update_doc.get("active_file_locks").is_some(), "{update_doc:?}"); + + let human = Command::new(bin()).args(["risk-map", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let human_text = String::from_utf8_lossy(&human.stdout); + assert!(human_text.contains("Diagnostics"), "{human_text:?}"); + assert!(human_text.contains("Risk map:"), "{human_text:?}"); + + let help = Command::new(bin()).args(["churn", "--help"]).output().unwrap(); + assert!(help.status.success()); + assert!(String::from_utf8_lossy(&help.stdout).contains("--unit"), "{}", String::from_utf8_lossy(&help.stdout)); +} + +#[test] +fn locks_show_on_the_risk_map_and_can_be_filtered() { + let dir = scratch("locks"); + write_sample(&dir); + init_repo(&dir); + commit_all(&dir, "init"); + let store = dir.join("store"); + assert!(Command::new(bin()).args(["--json", "analyze", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).status().unwrap().success()); + assert!(Command::new(bin()).args(["--json", "acquire-lock", "py/a.py", "agentX", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).status().unwrap().success()); + let risk = Command::new(bin()).args(["--json", "risk-map", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let doc = chisel::json::parse(String::from_utf8_lossy(&risk.stdout).trim()).unwrap(); + let locked = doc.get("files").and_then(|v| v.as_array()).unwrap().iter().find(|f| f.get("file_path").and_then(|v| v.as_str()) == Some("py/a.py")).unwrap(); + assert_eq!(locked.get("locked_by").and_then(|v| v.as_str()), Some("agentX")); + assert!(doc.get("_meta").and_then(|m| m.get("active_locks")).and_then(|v| v.as_i64()).unwrap_or(0) >= 1); + let listed = Command::new(bin()).args(["--json", "list-locks", "--agent-id", "agentX", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + assert!(listed.status.success(), "{}", String::from_utf8_lossy(&listed.stderr)); + assert!(String::from_utf8_lossy(&listed.stdout).contains("agentX")); + let other = Command::new(bin()).args(["--json", "list-locks", "--agent-id", "nobody", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).output().unwrap(); + let other_doc = chisel::json::parse(String::from_utf8_lossy(&other.stdout).trim()).unwrap(); + assert_eq!(other_doc.get("total").and_then(|v| v.as_i64()), Some(0), "{other_doc:?}"); +} + +#[test] +fn pytest_collection_noise_is_not_a_result_and_jest_paths_are_relative() { + assert!(chisel::run_cmd::parse_pytest_line("_____________________ ERROR collecting tests/x.py _____________________").is_none()); + let row = chisel::run_cmd::parse_pytest_line("tests/x.py::test_add PASSED [ 50%]").unwrap(); + assert_eq!(row.0, "tests/x.py::test_add"); + assert!(row.1); + let jest = r#"{"testResults":[{"name":"/tmp/sample/js/math.test.js","assertionResults":[{"title":"multiplies","status":"passed"}]}]}"#; + let rows = chisel::run_cmd::parse_jest_json(jest, Some(Path::new("/tmp/sample"))); + assert_eq!(rows[0].0, "js/math.test.js:multiplies"); +} + +#[test] +fn legacy_sqlite_results_migrate_into_the_store() { + let dir = scratch("migrate"); + fs::write(dir.join("a.py"), "def a():\n return 1\n").unwrap(); + init_repo(&dir); + commit_all(&dir, "init"); + let store = dir.join("store"); + let fixture = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/legacy/chisel.sqlite"); + let out = Command::new(bin()).args(["--json", "migrate", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).arg(&fixture).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let doc = chisel::json::parse(String::from_utf8_lossy(&out.stdout).trim()).unwrap(); + assert_eq!(doc.get("imported").and_then(|v| v.as_i64()), Some(3), "{doc:?}"); + let snap = fs::read_to_string(store.join("store")).unwrap(); + assert!(snap.contains("test_add"), "{snap:?}"); + assert!(snap.contains("duration_ms"), "{snap:?}"); + let again = Command::new(bin()).args(["--json", "migrate", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).arg(&fixture).output().unwrap(); + let second = chisel::json::parse(String::from_utf8_lossy(&again.stdout).trim()).unwrap(); + assert_eq!(second.get("imported").and_then(|v| v.as_i64()), Some(0), "{second:?}"); + assert_eq!(second.get("skipped").and_then(|v| v.as_i64()), Some(3), "{second:?}"); +} + +#[cfg(unix)] +#[test] +fn dep_extractor_hook_replaces_register_dep_extractor() { + let dir = scratch("dep"); + fs::create_dir_all(dir.join("src")).unwrap(); + fs::write(dir.join("src/app.py"), "def app():\n return 1\n").unwrap(); + fs::create_dir_all(dir.join("tests")).unwrap(); + fs::write(dir.join("tests/test_app.py"), "def test_app():\n return 1\n").unwrap(); + let hook = dir.join("dep-hook"); + fs::write(&hook, "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' '[{\"name\":\"app\",\"dep_type\":\"call\",\"confidence\":1}]'\n").unwrap(); + let _ = Command::new("chmod").arg("+x").arg(&hook).status(); + init_repo(&dir); + commit_all(&dir, "init"); + let store = dir.join("store"); + let out = Command::new(bin()) + .args(["--json", "analyze", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store) + .env("CHISEL_DEP_EXTRACTOR", &hook) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let snap = fs::read_to_string(store.join("store")).unwrap(); + assert!(snap.contains("src/app.py:app:function"), "custom dep extractor did not link app: {snap:?}"); +} + +#[test] +fn http_suggest_warns_on_eval() { + let dir = scratch("eval"); + fs::write(dir.join("dyn.js"), "function make(){ return eval('1'); }\n").unwrap(); + init_repo(&dir); + commit_all(&dir, "init"); + let store = dir.join("store"); + assert!(Command::new(bin()).args(["--json", "analyze", "--project-dir"]).arg(&dir).arg("--storage-dir").arg(&store).status().unwrap().success()); + let mut engine = chisel::Engine::open(&dir, Some(&store)).unwrap(); + let result = engine.dispatch("suggest_tests", &chisel::json::Value::object(vec![ + ("file_path".into(), chisel::json::Value::String("dyn.js".into())), + ])).unwrap(); + let warning = chisel::next_steps::eval_warning(&dir, Some("dyn.js")).unwrap(); + assert_eq!(warning.get("action").and_then(|v| v.as_str()), Some("warn_eval_used")); + let _ = result; +} From fca75c2d827e42901d5b32b19b748f95efbbeda8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 3 Nov 2026 13:05:34 +0000 Subject: [PATCH 2/2] Speed up large-tree analyse and queries past Python. Edge building and test-gaps no longer scan every unit for every import or edge. Snapshot loads skip an unchanged file and share one buffer for unescaped strings. Warm analyse and update skip unchanged files. Python method detection now sees defs after triple-quoted strings, so the large tree matches Python's 829 test units. Co-authored-by: Aron Amos --- README.md | 28 ++-- scripts/benchmark_results.md | 56 ++++---- src/engine/analyze.rs | 266 +++++++++++++++++++++++++---------- src/engine/mod.rs | 65 ++++++++- src/engine/query.rs | 144 +++++++++++-------- src/git.rs | 4 + src/http.rs | 5 +- src/impact.rs | 4 +- src/json.rs | 191 ++++++++++++++++++++----- src/store.rs | 152 ++++++++++++-------- src/syntax.rs | 151 +++++++++++++++----- 11 files changed, 747 insertions(+), 319 deletions(-) diff --git a/README.md b/README.md index 061d57d..730083f 100644 --- a/README.md +++ b/README.md @@ -189,20 +189,20 @@ Timed on this machine: Linux 6.12.94+, 4 vCPUs, Intel Xeon, 15.6 GiB RAM, Rust 1 | Command | Python small | Rust small | Python large | Rust large | |---|---:|---:|---:|---:| -| startup (`--help`) | 66.4 (64.9–74.1) | 0.3 (0.1–0.4) | 63.5 (62.6–72.3) | 0.4 (0.2–0.6) | -| cold analyse | 99.7 (96.7–118.2) | 7.5 (6.5–9.1) | 15088.1 (14562.3–15841.0) | 22393.5 (22178.9–22697.7) | -| warm analyse | 87.3 (86.6–96.7) | 7.8 (7.1–8.3) | 10358.8 (9935.4–10529.8) | 22332.3 (21923.2–22425.7) | -| warm update | 76.1 (71.1–83.1) | 2.7 (2.6–3.3) | 177.5 (163.3–215.0) | 190.9 (185.5–198.2) | -| impact | 67.3 (66.1–103.7) | 1.0 (0.8–1.1) | 125.3 (122.1–131.5) | 274.4 (255.0–281.6) | -| suggest-tests | 68.0 (65.5–98.9) | 1.0 (0.9–1.0) | 152.9 (146.3–155.2) | 263.7 (256.5–274.1) | -| risk-map | 69.2 (67.5–73.2) | 0.9 (0.8–0.9) | 283.7 (262.6–396.6) | 322.2 (313.5–416.8) | -| test-gaps | 69.6 (68.3–77.0) | 0.8 (0.7–1.0) | 210.6 (200.2–219.2) | 2815.4 (2720.7–3056.8) | -| triage | 66.9 (65.7–68.0) | 0.9 (0.9–1.0) | 293.4 (277.1–323.2) | 3091.5 (2908.0–3208.2) | -| churn | 65.5 (64.5–66.1) | 0.9 (0.8–1.0) | 80.4 (74.9–81.6) | 286.9 (268.0–294.8) | -| coupling | 67.5 (65.6–72.6) | 0.9 (0.8–1.0) | 85.8 (80.2–88.6) | 286.6 (260.3–334.0) | -| stats | 69.1 (65.1–71.6) | 0.8 (0.7–0.9) | 91.1 (78.9–97.2) | 264.3 (235.2–306.4) | - -On that large tree the Python snapshot had 829 test units and the Rust snapshot had 822. The small sample had 1 test unit on both sides. +| startup (`--help`) | 64.6 (63.4–73.4) | 0.3 (0.3–0.5) | 71.6 (64.7–76.1) | 0.4 (0.2–0.5) | +| cold analyse | 88.5 (86.7–93.9) | 8.0 (7.7–10.8) | 13692.5 (13542.5–14014.5) | 8034.2 (7627.3–8398.0) | +| warm analyse | 82.8 (80.7–89.8) | 2.3 (2.2–2.5) | 8646.5 (8604.2–8754.3) | 63.0 (61.3–77.9) | +| warm update | 71.0 (67.9–74.6) | 2.6 (2.2–2.6) | 146.4 (139.1–163.6) | 68.8 (65.3–71.2) | +| impact | 63.6 (63.0–66.2) | 0.6 (0.6–0.7) | 111.9 (106.5–113.3) | 45.4 (44.6–48.2) | +| suggest-tests | 64.3 (63.7–68.1) | 0.6 (0.6–0.7) | 125.9 (121.5–131.0) | 46.0 (44.6–51.9) | +| risk-map | 67.8 (65.9–75.4) | 0.7 (0.6–0.9) | 253.6 (233.9–261.0) | 57.5 (55.8–64.0) | +| test-gaps | 64.9 (63.2–69.7) | 0.6 (0.5–0.6) | 163.4 (161.9–180.5) | 73.1 (65.0–76.5) | +| triage | 65.8 (64.0–67.1) | 0.7 (0.6–0.8) | 254.4 (239.9–262.7) | 94.9 (93.3–97.5) | +| churn | 64.2 (62.9–66.6) | 0.6 (0.6–0.7) | 65.2 (64.4–69.0) | 48.8 (46.0–51.4) | +| coupling | 64.3 (63.3–65.5) | 0.6 (0.6–0.7) | 64.1 (63.3–66.1) | 47.7 (41.8–53.4) | +| stats | 64.8 (63.8–66.3) | 0.6 (0.5–0.6) | 67.4 (66.5–68.6) | 46.0 (45.0–51.0) | + +On that large tree both snapshots have 829 test units, and the small sample has 1 on both sides. Rust is faster on every command in both trees. ## License diff --git a/scripts/benchmark_results.md b/scripts/benchmark_results.md index 2e5df58..c0de6e8 100644 --- a/scripts/benchmark_results.md +++ b/scripts/benchmark_results.md @@ -18,39 +18,39 @@ Rust binary: `/agent/repos/Chisel/target/release/chisel` Python binary: `/tmp/chisel-py` Runs per command: 5 -Times are median milliseconds, with the minimum and maximum in parentheses. Peak RSS is the highest `VmHWM` sampled from `/proc//status` during each run, in KiB. +Times are median milliseconds, with the minimum and maximum in parentheses. Peak RSS is the highest VmHWM sampled from /proc during each run, in KiB. | Command | Python small | Rust small | Python large | Rust large | |---|---:|---:|---:|---:| -| startup | 66.4 (64.9–74.1) | 0.3 (0.1–0.4) | 63.5 (62.6–72.3) | 0.4 (0.2–0.6) | -| cold_analyze | 99.7 (96.7–118.2) | 7.5 (6.5–9.1) | 15088.1 (14562.3–15841.0) | 22393.5 (22178.9–22697.7) | -| warm_analyze | 87.3 (86.6–96.7) | 7.8 (7.1–8.3) | 10358.8 (9935.4–10529.8) | 22332.3 (21923.2–22425.7) | -| warm_update | 76.1 (71.1–83.1) | 2.7 (2.6–3.3) | 177.5 (163.3–215.0) | 190.9 (185.5–198.2) | -| impact | 67.3 (66.1–103.7) | 1.0 (0.8–1.1) | 125.3 (122.1–131.5) | 274.4 (255.0–281.6) | -| suggest_tests | 68.0 (65.5–98.9) | 1.0 (0.9–1.0) | 152.9 (146.3–155.2) | 263.7 (256.5–274.1) | -| risk_map | 69.2 (67.5–73.2) | 0.9 (0.8–0.9) | 283.7 (262.6–396.6) | 322.2 (313.5–416.8) | -| test_gaps | 69.6 (68.3–77.0) | 0.8 (0.7–1.0) | 210.6 (200.2–219.2) | 2815.4 (2720.7–3056.8) | -| triage | 66.9 (65.7–68.0) | 0.9 (0.9–1.0) | 293.4 (277.1–323.2) | 3091.5 (2908.0–3208.2) | -| churn | 65.5 (64.5–66.1) | 0.9 (0.8–1.0) | 80.4 (74.9–81.6) | 286.9 (268.0–294.8) | -| coupling | 67.5 (65.6–72.6) | 0.9 (0.8–1.0) | 85.8 (80.2–88.6) | 286.6 (260.3–334.0) | -| stats | 69.1 (65.1–71.6) | 0.8 (0.7–0.9) | 91.1 (78.9–97.2) | 264.3 (235.2–306.4) | +| startup | 64.6 (63.4–73.4) | 0.3 (0.3–0.5) | 71.6 (64.7–76.1) | 0.4 (0.2–0.5) | +| cold_analyze | 88.5 (86.7–93.9) | 8.0 (7.7–10.8) | 13692.5 (13542.5–14014.5) | 8034.2 (7627.3–8398.0) | +| warm_analyze | 82.8 (80.7–89.8) | 2.3 (2.2–2.5) | 8646.5 (8604.2–8754.3) | 63.0 (61.3–77.9) | +| warm_update | 71.0 (67.9–74.6) | 2.6 (2.2–2.6) | 146.4 (139.1–163.6) | 68.8 (65.3–71.2) | +| impact | 63.6 (63.0–66.2) | 0.6 (0.6–0.7) | 111.9 (106.5–113.3) | 45.4 (44.6–48.2) | +| suggest_tests | 64.3 (63.7–68.1) | 0.6 (0.6–0.7) | 125.9 (121.5–131.0) | 46.0 (44.6–51.9) | +| risk_map | 67.8 (65.9–75.4) | 0.7 (0.6–0.9) | 253.6 (233.9–261.0) | 57.5 (55.8–64.0) | +| test_gaps | 64.9 (63.2–69.7) | 0.6 (0.5–0.6) | 163.4 (161.9–180.5) | 73.1 (65.0–76.5) | +| triage | 65.8 (64.0–67.1) | 0.7 (0.6–0.8) | 254.4 (239.9–262.7) | 94.9 (93.3–97.5) | +| churn | 64.2 (62.9–66.6) | 0.6 (0.6–0.7) | 65.2 (64.4–69.0) | 48.8 (46.0–51.4) | +| coupling | 64.3 (63.3–65.5) | 0.6 (0.6–0.7) | 64.1 (63.3–66.1) | 47.7 (41.8–53.4) | +| stats | 64.8 (63.8–66.3) | 0.6 (0.5–0.6) | 67.4 (66.5–68.6) | 46.0 (45.0–51.0) | ## Peak RSS (KiB) | Command | Python small | Rust small | Python large | Rust large | |---|---:|---:|---:|---:| -| startup | 23028 | 1084 | 22980 | 2228 | -| cold_analyze | 24508 | 2492 | 60740 | 84112 | -| warm_analyze | 24268 | 608 | 54708 | 130340 | -| warm_update | 23932 | 2048 | 27244 | 71280 | -| impact | 23700 | 2044 | 28124 | 107248 | -| suggest_tests | 23720 | 1624 | 34844 | 107328 | -| risk_map | 23772 | 476 | 52652 | 107284 | -| test_gaps | 23764 | 476 | 59708 | 107244 | -| triage | 23796 | 484 | 52564 | 108756 | -| churn | 23740 | 2252 | 23656 | 107324 | -| coupling | 23672 | 1792 | 23724 | 107308 | -| stats | 23692 | 1628 | 25800 | 107244 | +| startup | 23056 | 2196 | 23040 | 2860 | +| cold_analyze | 24504 | 4592 | 60808 | 96600 | +| warm_analyze | 24284 | 672 | 54716 | 40604 | +| warm_update | 24024 | 2184 | 27200 | 40628 | +| impact | 23672 | 476 | 28264 | 40656 | +| suggest_tests | 23748 | 388 | 34932 | 40636 | +| risk_map | 23808 | 600 | 52540 | 40592 | +| test_gaps | 23796 | 1120 | 59680 | 46208 | +| triage | 23884 | 420 | 52552 | 49488 | +| churn | 23728 | 672 | 23752 | 40648 | +| coupling | 23684 | 588 | 23656 | 40664 | +| stats | 23732 | 912 | 25844 | 40644 | ## Failures @@ -58,9 +58,11 @@ None. ## Snapshot counts after the timed runs -- small rust: rust snapshot bytes 3923, test_units mentions 1 +- small rust: rust snapshot bytes 4044, test_units mentions 1 - small python: python test_units 1, test_edges 2 -- large rust: rust snapshot bytes 8129361, test_units mentions 822 +- large rust: rust snapshot bytes 10463225, test_units mentions 829 - large python: python test_units 829, test_edges 10267 The small repo is a two-file Python sample created by this script. The large repo is the django package from django/django (about 1000 code files) plus 80 of its test modules. History is 38 commits of at most 100 paths each: a shallow clone's boundary commit lists every file, and pairing those files dominates co-change time, so the benchmark uses this bounded history of the same tree. Counts are taken from the snapshot left by the last timed command. + +Both large snapshots have 829 test units, and the `(file, name)` pairs match. The earlier Rust count of 822 missed eight methods in `tests/migrations/test_multidb.py` (a triple-quoted SQL string containing parentheses hid every later `def`) and added one placeholder for `tests/bash_completion/management/commands/test_command.py`, which is a Django management command with no test functions. Python's extra seven were real tests, not false positives. diff --git a/src/engine/analyze.rs b/src/engine/analyze.rs index 6df5022..65ef2e9 100644 --- a/src/engine/analyze.rs +++ b/src/engine/analyze.rs @@ -5,11 +5,91 @@ use crate::hash::sha256_hex; use crate::impact::{self}; use crate::json::Value; use crate::store::{self}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +struct HashedFile { + hash: String, + mtime: f64, + mtime_ns: i64, + size: i64, +} + +impl HashedFile { + fn from_path(path: &std::path::Path, content: &[u8]) -> Self { + let meta = fs::metadata(path).ok(); + let mtime_ns = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_nanos() as i64).unwrap_or(0); + let mtime = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_secs_f64()).unwrap_or(0.0); + let size = meta.map(|m| m.len() as i64).unwrap_or(content.len() as i64); + Self { hash: sha256_hex(content), mtime, mtime_ns, size } + } + + fn row(&self, rel: &str) -> Value { + Value::object(vec![ + ("file_path".into(), Value::String(rel.to_string())), + ("hash".into(), Value::String(self.hash.clone())), + ("mtime".into(), Value::Float(self.mtime)), + ("mtime_ns".into(), Value::Int(self.mtime_ns)), + ("size".into(), Value::Int(self.size)), + ]) + } +} + +#[derive(Default)] +struct FileChurnAcc { + dates: Vec, + ins: i64, + del: i64, + authors: BTreeSet, + last: String, + count: i64, +} + +fn disk_mtime(path: &std::path::Path) -> (i64, i64, f64) { + let meta = fs::metadata(path).ok(); + let mtime_ns = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_nanos() as i64).unwrap_or(0); + let mtime = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_secs_f64()).unwrap_or(0.0); + let size = meta.map(|m| m.len() as i64).unwrap_or(-1); + (mtime_ns, size, mtime) +} + +fn disk_unchanged(path: &std::path::Path, mtime_ns: i64, size: i64, mtime: f64) -> bool { + let (cur_ns, cur_size, cur_mtime) = disk_mtime(path); + if cur_size != size { + return false; + } + if mtime_ns > 0 && cur_ns == mtime_ns { + return true; + } + mtime >= 0.0 && (cur_mtime - mtime).abs() <= 1e-3 +} + +fn hashes_match_disk(project: &std::path::Path, files: &[String], rows: &[Value]) -> bool { + if rows.len() != files.len() { + return false; + } + let mut have: HashMap<&str, (i64, i64, f64)> = HashMap::with_capacity(rows.len()); + for row in rows { + let Some(path) = row.get("file_path").and_then(|v| v.as_str()) else { return false }; + let mtime_ns = row.get("mtime_ns").and_then(|v| v.as_i64()).unwrap_or(0); + let size = row.get("size").and_then(|v| v.as_i64()).unwrap_or(-1); + let mtime = row.get("mtime").and_then(|v| v.as_f64()).unwrap_or(-1.0); + have.insert(path, (mtime_ns, size, mtime)); + } + if have.len() != files.len() { + return false; + } + for rel in files { + let Some((mtime_ns, size, mtime)) = have.get(rel.as_str()) else { return false }; + if !disk_unchanged(&project.join(rel), *mtime_ns, *size, *mtime) { + return false; + } + } + true +} + impl Engine { pub fn analyze(&mut self) -> Result { if !self.shards.is_empty() && self.only_files.is_none() { @@ -29,9 +109,13 @@ impl Engine { } let git = self.git.fork(); let files = self.scan_code_files(&git); + if let Some(summary) = self.cached_analyze(&git, &files) { + return Ok(summary); + } let commits = git.parse_log().unwrap_or_default(); let mut units_by_file: BTreeMap> = BTreeMap::new(); let mut all_units = Vec::new(); + let mut hashed: Vec = Vec::with_capacity(files.len()); for (index, rel) in files.iter().enumerate() { if let Some(id) = self.active_job.clone() { let pct = ((index as i64) * 100 / (files.len() as i64).max(1)).min(99); @@ -53,16 +137,12 @@ impl Engine { let content = fs::read(&abs).unwrap_or_default(); let text = String::from_utf8_lossy(&content); let units = extract::extract_code_units_in(rel, &text, Some(&self.project_dir), self.allow_repo_extractor); - let hash = sha256_hex(&content); + let hashed_file = HashedFile::from_path(&abs, &content); for u in &units { - all_units.push(unit_obj(rel, u, &hash)); + all_units.push(unit_obj(rel, u, &hashed_file.hash)); } units_by_file.insert(rel.clone(), units); - let meta = fs::metadata(&abs).ok(); - let mtime = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_secs_f64()).unwrap_or(0.0); - let size = meta.map(|m| m.len() as i64).unwrap_or(0); - // stored below - let _ = (mtime, size, hash); + hashed.push(hashed_file); } self.store.doc.insert("code_units", Value::Array(all_units)); @@ -92,28 +172,33 @@ impl Engine { let now = impact::now_unix(); let mut churn_rows = Vec::new(); - for rel in &files { - let dates: Vec = commits.iter().filter(|c| c.files.iter().any(|f| &f.path == rel)).map(|c| c.date.clone()).collect(); - let mut ins = 0i64; - let mut del = 0i64; - let mut authors = BTreeSet::new(); - let mut last = String::new(); - let mut count = 0i64; - for c in &commits { - if let Some(f) = c.files.iter().find(|f| &f.path == rel) { - if impact::parse_iso_unix(&c.date).is_some() { - count += 1; - authors.insert(c.author.clone()); - ins += f.insertions; - del += f.deletions; - if last.is_empty() || c.date > last { - last = c.date.clone(); - } + let mut file_churn: HashMap<&str, FileChurnAcc> = HashMap::new(); + for c in &commits { + let parsed = impact::parse_iso_unix(&c.date).is_some(); + let mut seen = HashSet::new(); + for f in &c.files { + if !seen.insert(f.path.as_str()) { + continue; + } + let acc = file_churn.entry(f.path.as_str()).or_default(); + acc.dates.push(c.date.clone()); + if parsed { + acc.count += 1; + acc.authors.insert(c.author.clone()); + acc.ins += f.insertions; + acc.del += f.deletions; + if acc.last.is_empty() || c.date > acc.last { + acc.last = c.date.clone(); } } } + } + for rel in &files { + let acc = file_churn.get(rel.as_str()); + let dates = acc.map(|a| a.dates.clone()).unwrap_or_default(); let score = impact::churn_score(&dates, now); - churn_rows.push(churn_row(rel, "", ChurnCounts { commits: count, authors: authors.len() as i64, insertions: ins, deletions: del }, &last, score)); + let (count, authors, ins, del, last) = acc.map(|a| (a.count, a.authors.len() as i64, a.ins, a.del, a.last.clone())).unwrap_or((0, 0, 0, 0, String::new())); + churn_rows.push(churn_row(rel, "", ChurnCounts { commits: count, authors, insertions: ins, deletions: del }, &last, score)); } if files.len() <= 2000 { let tasks = unit_log_tasks(&files, &units_by_file); @@ -141,27 +226,15 @@ impl Engine { let mut blame_rows = Vec::new(); let mut hash_rows = Vec::new(); - for rel in &files { - let abs = self.project_dir.join(rel); - let content = fs::read(&abs).unwrap_or_default(); - let hash = sha256_hex(&content); - let meta = fs::metadata(&abs).ok(); - let mtime = meta.as_ref().and_then(|m| m.modified().ok()).and_then(|t| t.duration_since(UNIX_EPOCH).ok()).map(|d| d.as_secs_f64()).unwrap_or(0.0); - let size = meta.map(|m| m.len() as i64).unwrap_or(0); - hash_rows.push(Value::object(vec![ - ("file_path".into(), Value::String(rel.clone())), - ("hash".into(), Value::String(hash.clone())), - ("mtime".into(), Value::Float(mtime)), - ("size".into(), Value::Int(size)), - ])); + for (rel, hashed_file) in files.iter().zip(&hashed) { + hash_rows.push(hashed_file.row(rel)); } let blame_blocks = pool_map(&files, |rel| git.blame(rel).unwrap_or_default()); - for (rel, blocks) in files.iter().zip(blame_blocks) { - let hash = hash_rows.iter().find(|h| h.get("file_path").and_then(|v| v.as_str()) == Some(rel.as_str())).and_then(|h| h.get("hash").and_then(|v| v.as_str())).unwrap_or(""); + for ((rel, blocks), hashed_file) in files.iter().zip(blame_blocks).zip(&hashed) { for b in blocks { blame_rows.push(Value::object(vec![ ("file_path".into(), Value::String(rel.clone())), - ("content_hash".into(), Value::String(hash.into())), + ("content_hash".into(), Value::String(hashed_file.hash.clone())), ("line_start".into(), Value::Int(b.line_start as i64)), ("line_end".into(), Value::Int(b.line_end as i64)), ("author".into(), Value::String(b.author)), @@ -180,31 +253,52 @@ impl Engine { let imports = self.build_import_edges(&files); self.store.doc.insert("import_edges", Value::Array(imports)); store::meta_set(&mut self.store.doc, "project_fingerprint", &self.project_dir.to_string_lossy()); - - let test_files = self.test_file_set().len() as i64; - let test_units_n = self.store.array("test_units").len() as i64; - let edge_n = self.store.array("test_edges").len() as i64; - let unit_n = self.store.array("code_units").len() as i64; if let Some(latest) = commits.iter().map(|c| c.date.as_str()).max() { store::meta_set(&mut self.store.doc, "last_commit_date", latest); } + if let Ok(head) = git.head_sha() { + store::meta_set(&mut self.store.doc, "analyzed_head", &head); + } self.store.save()?; + Ok(self.analyze_summary(files.len())) + } + + /// Second `analyze` with the same tree and HEAD returns the stored summary. + fn cached_analyze(&self, git: &GitAnalyzer, files: &[String]) -> Option { + if self.active_job.is_some() || self.only_files.is_some() { + return None; + } + if self.store.array_len("code_units") == 0 || self.store.array_len("file_hashes") == 0 { + return None; + } + if !hashes_match_disk(&self.project_dir, files, self.store.rows("file_hashes")) { + return None; + } + let Ok(head) = git.head_sha() else { return None }; + if store::meta_get(&self.store.doc, "analyzed_head").as_deref() != Some(head.as_str()) { + return None; + } + Some(self.analyze_summary(files.len())) + } + + fn analyze_summary(&self, files_scanned: usize) -> Value { + let test_files = self.test_file_set().len() as i64; let mut summary = vec![ - ("code_files_scanned".into(), Value::Int(files.len() as i64)), - ("code_units_found".into(), Value::Int(unit_n)), + ("code_files_scanned".into(), Value::Int(files_scanned as i64)), + ("code_units_found".into(), Value::Int(self.store.array_len("code_units") as i64)), ("test_files_found".into(), Value::Int(test_files)), - ("test_units_found".into(), Value::Int(test_units_n)), - ("test_edges_built".into(), Value::Int(edge_n)), - ("commits_parsed".into(), Value::Int(commits.len() as i64)), + ("test_units_found".into(), Value::Int(self.store.array_len("test_units") as i64)), + ("test_edges_built".into(), Value::Int(self.store.array_len("test_edges") as i64)), + ("commits_parsed".into(), Value::Int(self.store.array_len("commits") as i64)), ("orphaned_results_cleaned".into(), Value::Int(0)), - ("active_file_locks".into(), Value::Int(self.store.array("file_locks").len() as i64)), + ("active_file_locks".into(), Value::Int(self.store.array_len("file_locks") as i64)), ("hint".into(), Value::String(analyze_hint(test_files))), ]; - if files.len() > 2000 { + if files_scanned > 2000 { summary.push(("unit_churn_skipped".into(), Value::Bool(true))); } push_extractor_warnings(&mut summary); - Ok(Value::object(summary)) + Value::object(summary) } pub fn update(&mut self) -> Result { @@ -214,21 +308,34 @@ impl Engine { extract::clear_extractor_warnings(); let git = self.git.fork(); let files = self.scan_code_files(&git); - let before: BTreeMap = self.store.array("file_hashes").into_iter().filter_map(|row| { - Some((row.get("file_path")?.as_str()?.to_string(), row.get("hash")?.as_str()?.to_string())) - }).collect(); + let mut before: HashMap = HashMap::new(); + for row in self.store.rows("file_hashes") { + let Some(path) = row.get("file_path").and_then(|v| v.as_str()) else { continue }; + let Some(hash) = row.get("hash").and_then(|v| v.as_str()) else { continue }; + let mtime_ns = row.get("mtime_ns").and_then(|v| v.as_i64()).unwrap_or(0); + let size = row.get("size").and_then(|v| v.as_i64()).unwrap_or(-1); + let mtime = row.get("mtime").and_then(|v| v.as_f64()).unwrap_or(-1.0); + before.insert(path.to_string(), (hash.to_string(), mtime_ns, size, mtime)); + } let mut changed: Vec = Vec::new(); for rel in &files { let abs = self.project_dir.join(rel); + if let Some((hash, mtime_ns, size, mtime)) = before.get(rel) { + if disk_unchanged(&abs, *mtime_ns, *size, *mtime) { + let _ = hash; + continue; + } + } let content = fs::read(&abs).unwrap_or_default(); let hash = sha256_hex(&content); - if before.get(rel).map(|h| h.as_str()) != Some(hash.as_str()) { + if before.get(rel).map(|(h, _, _, _)| h.as_str()) != Some(hash.as_str()) { changed.push(rel.clone()); } } - let removed: Vec = before.keys().filter(|path| !files.iter().any(|f| f == *path)).cloned().collect(); + let file_set: HashSet<&str> = files.iter().map(|s| s.as_str()).collect(); + let removed: Vec = before.keys().filter(|path| !file_set.contains(path.as_str())).cloned().collect(); let since = store::meta_get(&self.store.doc, "last_commit_date"); - let known: BTreeSet = self.store.array("commits").iter().filter_map(|c| c.get("hash").and_then(|v| v.as_str()).map(|s| s.to_string())).collect(); + let known: BTreeSet = self.store.rows("commits").iter().filter_map(|c| c.get("hash").and_then(|v| v.as_str()).map(|s| s.to_string())).collect(); let new_commits: Vec = git.parse_log_since(since.as_deref()).unwrap_or_default().into_iter().filter(|c| !known.contains(&c.hash)).collect(); let commit_touch: BTreeSet = new_commits.iter().flat_map(|c| c.files.iter().map(|f| f.path.clone())).collect(); if changed.is_empty() && new_commits.is_empty() && removed.is_empty() { @@ -392,13 +499,7 @@ impl Engine { for rel in &scope { let abs = self.project_dir.join(rel); let content = fs::read(&abs).unwrap_or_default(); - let hash = sha256_hex(&content); - hash_rows.push(Value::object(vec![ - ("file_path".into(), Value::String(rel.clone())), - ("hash".into(), Value::String(hash.clone())), - ("mtime".into(), Value::Float(0.0)), - ("size".into(), Value::Int(content.len() as i64)), - ])); + hash_rows.push(HashedFile::from_path(&abs, &content).row(rel)); } let blame_blocks = pool_map(&scope, |rel| git.blame(rel).unwrap_or_default()); for (rel, blocks) in scope.iter().zip(blame_blocks) { @@ -590,7 +691,7 @@ impl Engine { } let hash = sha256_hex(content.as_bytes()); if selected.is_empty() { - if matches!(framework, "junit" | "csharp_test" | "xctest" | "gtest") { + if matches!(framework, "junit" | "csharp_test" | "xctest" | "gtest" | "pytest" | "go" | "jest" | "playwright") { continue; } let stem = rel.rsplit(['/', '\\']).next().unwrap_or(rel); @@ -622,21 +723,30 @@ impl Engine { let mut static_rows = Vec::new(); let mut static_seen = BTreeSet::new(); let mut seen = BTreeSet::new(); - let mut content_cache: BTreeMap = BTreeMap::new(); let shared_index = bare_name_index(&code_units); if !test_units.is_empty() { self.bare_index_builds += 1; } + let import_index = ImportPathIndex::from_units(&code_units); + let mut id_file: HashMap<&str, &str> = HashMap::new(); + for unit in &code_units { + if let (Some(id), Some(path)) = (unit.get("id").and_then(|v| v.as_str()), unit.get("file_path").and_then(|v| v.as_str())) { + id_file.insert(id, path); + } + } + let mut dep_cache: BTreeMap, BTreeSet)> = BTreeMap::new(); for tu in &test_units { let file = tu.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); let tid = tu.get("id").and_then(|v| v.as_str()).unwrap_or(""); let index = &shared_index; - let content = content_cache.entry(file.to_string()).or_insert_with(|| { - fs::read_to_string(self.project_dir.join(file)).unwrap_or_default() - }); - let deps = extract::extract_deps_in(file, content, Some(&self.project_dir), self.allow_repo_extractor); - let resolved = resolved_import_files(&deps, &code_units); - for path in &resolved { + if !dep_cache.contains_key(file) { + let content = fs::read_to_string(self.project_dir.join(file)).unwrap_or_default(); + let deps = extract::extract_deps_in(file, &content, Some(&self.project_dir), self.allow_repo_extractor); + let resolved = resolve_deps(&deps, &import_index); + dep_cache.insert(file.to_string(), (deps, resolved)); + } + let (deps, resolved) = dep_cache.get(file).expect("deps"); + for path in resolved { let key = format!("{file}|{path}"); if static_seen.insert(key) { static_rows.push(Value::object(vec![ @@ -646,9 +756,9 @@ impl Engine { } } for dep in deps { - let matched = match_dep(file, &dep, &code_units, &resolved, index); + let matched = match_dep(file, dep, &code_units, resolved, index); for cid in matched { - let code_file = code_units.iter().find(|u| u.get("id").and_then(|v| v.as_str()) == Some(cid.as_str())).and_then(|u| u.get("file_path").and_then(|v| v.as_str())).unwrap_or(""); + let code_file = id_file.get(cid.as_str()).copied().unwrap_or(""); if code_file.is_empty() || code_file == file { continue; } diff --git a/src/engine/mod.rs b/src/engine/mod.rs index e799d3f..518fe60 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -10,7 +10,7 @@ use crate::git::{self, GitAnalyzer}; use crate::impact; use crate::json::Value; use crate::store::{self, Store}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -169,6 +169,7 @@ pub(crate) fn argument_missing(args: &Value, field: &str, ty: &str) -> bool { match value { Value::Null => true, Value::String(text) if text.is_empty() => true, + Value::Shared(text) if text.as_str().is_empty() => true, Value::Array(items) if items.is_empty() => true, _ if ty == "boolean" => !matches!(value, Value::Bool(_)), _ => false, @@ -294,7 +295,7 @@ pub(crate) fn stale_status() -> Value { ]) } -pub(crate) fn push_extractor_warnings(summary: &mut Vec<(String, Value)>) { +pub(crate) fn push_extractor_warnings(summary: &mut Vec<(crate::json::SharedStr, Value)>) { let warnings = extract::take_extractor_warnings(); if warnings.is_empty() { return; @@ -554,15 +555,65 @@ fn rust_integration_test(path: &str, name: &str) -> bool { normalized.split('/').next() == Some("tests") } +/// Suffix map of code paths so an import resolves without scanning every unit. +pub(crate) struct ImportPathIndex { + by_suffix: HashMap>, +} + +impl ImportPathIndex { + pub(crate) fn from_units(units: &[Value]) -> Self { + let mut seen = BTreeSet::new(); + let mut paths = Vec::new(); + for unit in units { + if let Some(path) = unit.get("file_path").and_then(|v| v.as_str()) { + if seen.insert(path) { + paths.push(path.to_string()); + } + } + } + let mut by_suffix: HashMap> = HashMap::new(); + for path in &paths { + let normalized = path.replace('\\', "/"); + let mut rest = normalized.as_str(); + loop { + by_suffix.entry(rest.to_string()).or_default().push(path.clone()); + if let Some((_, tail)) = rest.split_once('/') { + rest = tail; + } else { + break; + } + } + } + Self { by_suffix } + } + + pub(crate) fn files_for_module(&self, module_path: &str) -> &[String] { + if module_path.is_empty() { + return &[]; + } + let mut key = String::with_capacity(module_path.len() + 3); + for c in module_path.chars() { + if c == '.' { + key.push('/'); + } else { + key.push(c); + } + } + key.push_str(".py"); + self.by_suffix.get(&key).map(|v| v.as_slice()).unwrap_or(&[]) + } +} + pub(crate) fn resolved_import_files(deps: &[Dep], units: &[Value]) -> BTreeSet { + resolve_deps(deps, &ImportPathIndex::from_units(units)) +} + +pub(crate) fn resolve_deps(deps: &[Dep], index: &ImportPathIndex) -> BTreeSet { let mut out = BTreeSet::new(); for dep in deps { let Some(module) = dep.module_path.as_deref() else { continue }; - for u in units { - let path = u.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); - if extract::matches_import_path(path, module) { - out.insert(path.to_string()); - } + for path in index.files_for_module(module) { + out.insert(path.clone()); } } out diff --git a/src/engine/query.rs b/src/engine/query.rs index 9129800..8563211 100644 --- a/src/engine/query.rs +++ b/src/engine/query.rs @@ -5,9 +5,18 @@ use crate::hash::sha256_hex; use crate::impact::{self, RiskInputs}; use crate::json::Value; use crate::store::{self, Store}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; +fn cached_exists(cache: &mut HashMap, project: &std::path::Path, path: &str) -> bool { + if let Some(hit) = cache.get(path) { + return *hit; + } + let ok = project.join(path).exists(); + cache.insert(path.to_string(), ok); + ok +} + fn cross_edge_key(edge: &Value) -> String { format!( "{}|{}|{}", @@ -19,7 +28,7 @@ fn cross_edge_key(edge: &Value) -> String { impl Engine { pub fn has_data(&self) -> bool { - !self.store.array("code_units").is_empty() + self.store.array_len("code_units") > 0 } pub(super) fn no_data() -> Value { @@ -44,11 +53,11 @@ impl Engine { if !self.has_data() { return Ok(Self::no_data()); } - let gtest_files: BTreeSet = self.store.array("test_units").into_iter().filter_map(|u| { + let gtest_files: BTreeSet = self.store.rows("test_units").iter().filter_map(|u| { let framework = u.get("framework").and_then(|v| v.as_str())?; if framework == "gtest" { u.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string()) } else { None } }).collect(); - let mut files: Vec = self.store.array("churn").into_iter().filter_map(|c| c.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).filter(|p| { + let mut files: Vec = self.store.rows("churn").iter().filter_map(|c| c.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).filter(|p| { if let Some(dir) = &self.directory { if !path_in_directory(p, dir) { return false; @@ -62,10 +71,10 @@ impl Engine { files.sort(); files.dedup(); let query_min: i64 = store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(1); - let imports = self.store.array("import_edges"); + let imports = self.store.rows("import_edges"); let mut neighbor_map: BTreeMap> = BTreeMap::new(); - for e in &imports { + for e in imports { let a = e.get("importer_file").and_then(|v| v.as_str()).unwrap_or("").to_string(); let b = e.get("imported_file").and_then(|v| v.as_str()).unwrap_or("").to_string(); let conf = e.get("confidence").and_then(|v| v.as_f64()).unwrap_or(1.0); @@ -87,7 +96,7 @@ impl Engine { let file_set: BTreeSet = files.iter().cloned().collect(); let hops = impact::hops_to_tested(&file_set, &tested_files, &neighbor_map); let mut branch_counts: BTreeMap = BTreeMap::new(); - for row in self.store.array("branch_co_changes") { + for row in self.store.rows("branch_co_changes") { let n = row.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0); if n < 1 { continue; @@ -106,6 +115,7 @@ impl Engine { let failure_rates = failure_rate_map(&recorded); let duration_cvs = duration_cv_map(&recorded); let trend_now = now_secs(); + let mut exists_cache: HashMap = HashMap::new(); let mut scored = Vec::new(); for f in &files { let churn_raw = self.store.churn_for_file(f).into_iter().find(|c| c.get("unit_name").and_then(|v| v.as_str()).unwrap_or("").is_empty()).and_then(|c| c.get("churn_score").and_then(|v| v.as_f64())).unwrap_or(0.0); @@ -123,7 +133,7 @@ impl Engine { continue; } let other = if a == f { b } else { a }; - if !self.project_dir.join(other).exists() { + if !cached_exists(&mut exists_cache, &self.project_dir, other) { continue; } co_count += 1; @@ -134,9 +144,9 @@ impl Engine { } } partners.sort_by(|a, b| b.1.cmp(&a.1).then(a.2.cmp(&b.2)).then(a.0.cmp(&b.0))); - let mut import_neighbors: Vec = neighbor_map.get(f).map(|s| s.iter().filter(|n| self.project_dir.join(n.as_str()).exists()).cloned().collect()).unwrap_or_default(); + let mut import_neighbors: Vec = neighbor_map.get(f).map(|s| s.iter().filter(|n| cached_exists(&mut exists_cache, &self.project_dir, n)).cloned().collect()).unwrap_or_default(); import_neighbors.sort(); - let file_units: Vec = self.store.units_by_file(f).into_iter().filter(|u| !is_test_unit_row(u)).collect(); + let file_units: Vec<&Value> = self.store.units_by_file(f).into_iter().filter(|u| !is_test_unit_row(u)).collect(); let mut tested_lines = 0i64; let mut total_lines = 0i64; let mut tested_units = 0i64; @@ -236,11 +246,11 @@ impl Engine { } else { Value::object(vec![("reweighted".into(), Value::Bool(false))]) }; - let commits_n = self.store.array("commits").len() as i64; + let commits_n = self.store.array_len("commits") as i64; let test_edges_n = self.store.array_len("test_edges") as i64; let test_results_n = recorded.len() as i64; let import_n = imports.len() as i64; - let co_n = self.store.array("co_changes").iter().filter(|c| c.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0) >= query_min).count() as i64; + let co_n = self.store.rows("co_changes").iter().filter(|c| c.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0) >= query_min).count() as i64; let mut meta = impact::build_risk_meta(&scored, commits_n, test_edges_n, test_results_n, import_n, co_n); if let Some(obj) = meta.as_object_mut() { // merge reweighting keys @@ -259,7 +269,7 @@ impl Engine { let scored_paths: Vec = scored.iter().filter_map(|row| row.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string())).collect(); let scored_set: BTreeSet = scored_paths.iter().cloned().collect(); let mut directed: BTreeMap> = BTreeMap::new(); - for edge in &imports { + for edge in imports { let conf = edge.get("confidence").and_then(|v| v.as_f64()).unwrap_or(1.0); if conf < 1.0 { continue; @@ -477,9 +487,9 @@ impl Engine { } pub(super) fn static_hits(&mut self, file: &str) -> Vec { - let tests = self.store.array("test_units"); + let tests = self.store.rows("test_units"); let mut by_file: BTreeMap> = BTreeMap::new(); - for t in &tests { + for t in tests { let fp = t.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); let name = t.get("name").and_then(|v| v.as_str()).unwrap_or(""); by_file.entry(fp.to_string()).or_default().push(name.to_string()); @@ -494,7 +504,7 @@ impl Engine { targets_by_test.insert(fp.clone(), targets); } } else { - for row in self.store.array("static_test_imports") { + for row in self.store.rows("static_test_imports") { let test_file = row.get("test_file").and_then(|v| v.as_str()).unwrap_or(""); let code_file = row.get("code_file").and_then(|v| v.as_str()).unwrap_or(""); if test_file.is_empty() || code_file.is_empty() { continue; } @@ -643,9 +653,8 @@ impl Engine { } } let query_min: i64 = store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(1); - let co = self.store.array("co_changes"); for file in files { - for cc in &co { + for cc in self.store.co_for_file(file) { let a = cc.get("file_a").and_then(|v| v.as_str()).unwrap_or(""); let b = cc.get("file_b").and_then(|v| v.as_str()).unwrap_or(""); let n = cc.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0); @@ -666,9 +675,9 @@ impl Engine { } } } - let imports = self.store.array("import_edges"); + let imports = self.store.rows("import_edges"); let mut reverse: BTreeMap> = BTreeMap::new(); - for e in &imports { + for e in imports { let importer = e.get("importer_file").and_then(|v| v.as_str()).unwrap_or("").to_string(); let imported = e.get("imported_file").and_then(|v| v.as_str()).unwrap_or("").to_string(); let conf = e.get("confidence").and_then(|v| v.as_f64()).unwrap_or(1.0); @@ -854,31 +863,31 @@ impl Engine { } pub(super) fn stats_doc(&self, include_results: bool) -> Value { - let results = self.store.array("test_results"); + let results = self.store.rows("test_results"); let passed = results.iter().filter(|r| r.get("passed").and_then(|v| v.as_bool()) == Some(true)).count() as i64; let failed = results.iter().filter(|r| r.get("passed").and_then(|v| v.as_bool()) == Some(false)).count() as i64; let mut pairs = vec![ - ("code_units".into(), Value::Int(self.store.array("code_units").len() as i64)), - ("test_units".into(), Value::Int(self.store.array("test_units").len() as i64)), - ("test_edges".into(), Value::Int(self.store.array("test_edges").len() as i64)), - ("commits".into(), Value::Int(self.store.array("commits").len() as i64)), - ("co_changes".into(), Value::Int(self.store.array("co_changes").len() as i64)), - ("import_edges".into(), Value::Int(self.store.array("import_edges").len() as i64)), + ("code_units".into(), Value::Int(self.store.array_len("code_units") as i64)), + ("test_units".into(), Value::Int(self.store.array_len("test_units") as i64)), + ("test_edges".into(), Value::Int(self.store.array_len("test_edges") as i64)), + ("commits".into(), Value::Int(self.store.array_len("commits") as i64)), + ("co_changes".into(), Value::Int(self.store.array_len("co_changes") as i64)), + ("import_edges".into(), Value::Int(self.store.array_len("import_edges") as i64)), ("test_results".into(), Value::Int(results.len() as i64)), ("passed".into(), Value::Int(passed)), ("failed".into(), Value::Int(failed)), - ("commit_files".into(), Value::Int(self.store.array("commit_files").len() as i64)), - ("blame_cache".into(), Value::Int(self.store.array("blame").len() as i64)), - ("branch_co_changes".into(), Value::Int(self.store.array("branch_co_changes").len() as i64)), - ("churn_stats".into(), Value::Int(self.store.array("churn").len() as i64)), - ("file_hashes".into(), Value::Int(self.store.array("file_hashes").len() as i64)), - ("coupling_threshold".into(), Value::Int(impact::coupling_threshold(self.store.array("commits").len() as i64))), + ("commit_files".into(), Value::Int(self.store.array_len("commit_files") as i64)), + ("blame_cache".into(), Value::Int(self.store.array_len("blame") as i64)), + ("branch_co_changes".into(), Value::Int(self.store.array_len("branch_co_changes") as i64)), + ("churn_stats".into(), Value::Int(self.store.array_len("churn") as i64)), + ("file_hashes".into(), Value::Int(self.store.array_len("file_hashes") as i64)), + ("coupling_threshold".into(), Value::Int(impact::coupling_threshold(self.store.array_len("commits") as i64))), ("co_change_query_min".into(), Value::Int(store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(3).max(1))), ]; - let edge_rows = self.store.array("test_edges"); + let edge_rows = self.store.rows("test_edges"); if !edge_rows.is_empty() { let mut counts: BTreeMap<&str, i64> = BTreeMap::new(); - for edge in &edge_rows { + for edge in edge_rows { let kind = edge.get("edge_type").and_then(|v| v.as_str()).unwrap_or(""); *counts.entry(kind).or_default() += 1; } @@ -900,7 +909,7 @@ impl Engine { } } if include_results { - pairs.push(("recorded_results".into(), Value::Array(results))); + pairs.push(("recorded_results".into(), Value::Array(results.to_vec()))); } Value::object(pairs) } @@ -1544,36 +1553,46 @@ impl Engine { } pub(super) fn gap_rows(&self) -> Vec { - let edges = self.store.array("test_edges"); - let churn = self.store.array("churn"); - let gtest_files: BTreeSet = self.store.array("test_units").into_iter().filter_map(|u| { + let edges = self.store.rows("test_edges"); + let mut covered: BTreeSet<&str> = BTreeSet::new(); + for edge in edges { + if let Some(id) = edge.get("code_id").and_then(|v| v.as_str()) { + covered.insert(id); + } + } + let mut churn_by: BTreeMap<(&str, &str), (f64, i64)> = BTreeMap::new(); + for row in self.store.rows("churn") { + let path = row.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); + let unit = row.get("unit_name").and_then(|v| v.as_str()).unwrap_or(""); + let score = row.get("churn_score").and_then(|v| v.as_f64()).unwrap_or(0.0); + let commits = row.get("commit_count").and_then(|v| v.as_i64()).unwrap_or(0); + churn_by.insert((path, unit), (score, commits)); + } + let gtest_files: BTreeSet<&str> = self.store.rows("test_units").iter().filter_map(|u| { if u.get("framework").and_then(|v| v.as_str()) == Some("gtest") { - u.get("file_path").and_then(|v| v.as_str()).map(|s| s.to_string()) + u.get("file_path").and_then(|v| v.as_str()) } else { None } }).collect(); let mut gaps = Vec::new(); - for u in self.store.array("code_units") { + for u in self.store.rows("code_units") { let path = u.get("file_path").and_then(|v| v.as_str()).unwrap_or(""); if let Some(file) = &self.scope_file { if path != file.as_str() { continue; } } else if let Some(dir) = &self.directory { if !path_in_directory(path, dir) { continue; } } - if is_test_unit_row(&u) && self.exclude_tests { continue; } + if is_test_unit_row(u) && self.exclude_tests { continue; } if self.exclude_tests && (looks_like_test(path) || gtest_files.contains(path)) { continue; } let id = u.get("id").and_then(|v| v.as_str()).unwrap_or(""); - if edges.iter().any(|e| e.get("code_id").and_then(|v| v.as_str()) == Some(id)) { + if covered.contains(id) { continue; } let name = u.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let stat = churn.iter().find(|c| { - c.get("file_path").and_then(|v| v.as_str()) == Some(path) - && c.get("unit_name").and_then(|v| v.as_str()).unwrap_or("") == name - }); - let churn_score = stat.and_then(|c| c.get("churn_score")).and_then(|v| v.as_f64()).unwrap_or(0.0); - let commit_count = stat.and_then(|c| c.get("commit_count")).and_then(|v| v.as_i64()).unwrap_or(0); + let stat = churn_by.get(&(path, name)); + let churn_score = stat.map(|(score, _)| *score).unwrap_or(0.0); + let commit_count = stat.map(|(_, commits)| *commits).unwrap_or(0); gaps.push(Value::object(vec![ ("id".into(), Value::String(id.into())), ("file_path".into(), Value::String(path.into())), @@ -1699,7 +1718,7 @@ impl Engine { stale.truncate(cap); truncated = true; } - let commits_n = self.store.array("commits").len() as i64; + let commits_n = self.store.array_len("commits") as i64; let mut body = vec![ ("top_risk_files".into(), Value::Array(top.clone())), ("test_gaps".into(), Value::Array(gaps.clone())), @@ -1708,8 +1727,8 @@ impl Engine { ("files_triaged".into(), Value::Int(top.len() as i64)), ("total_test_gaps".into(), Value::Int(gaps.len() as i64)), ("total_stale_tests".into(), Value::Int(stale.len() as i64)), - ("test_edge_count".into(), Value::Int(self.store.array("test_edges").len() as i64)), - ("test_result_count".into(), Value::Int(self.store.array("test_results").len() as i64)), + ("test_edge_count".into(), Value::Int(self.store.array_len("test_edges") as i64)), + ("test_result_count".into(), Value::Int(self.store.array_len("test_results") as i64)), ("coupling_threshold".into(), Value::Int(impact::coupling_threshold(commits_n))), ("active_locks".into(), Value::Int(self.active_locks().len() as i64)), ])), @@ -1731,7 +1750,7 @@ impl Engine { match name { "churn" => { let unit = args.get("unit_name").and_then(|v| v.as_str()).filter(|s| !s.is_empty()); - let mut rows: Vec = self.store.array("churn").into_iter().filter(|c| c.get("file_path").and_then(|v| v.as_str()) == Some(file)).collect(); + let mut rows: Vec = self.store.churn_for_file(file).into_iter().cloned().collect(); if let Some(unit) = unit { rows.retain(|c| c.get("unit_name").and_then(|v| v.as_str()).unwrap_or("") == unit); } else { @@ -1884,16 +1903,17 @@ impl Engine { let query_min: i64 = store::meta_get(&self.store.doc, "co_change_query_min").and_then(|s| s.parse().ok()).unwrap_or(3); let effective = min_count.max(query_min); let mut partners = Vec::new(); - for c in self.store.array("co_changes") { + let mut exists_cache: HashMap = HashMap::new(); + for c in self.store.co_for_file(file) { let n = c.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0); if n < effective { continue; } let a = c.get("file_a").and_then(|v| v.as_str()).unwrap_or(""); let b = c.get("file_b").and_then(|v| v.as_str()).unwrap_or(""); let other = if a == file { b } else if b == file { a } else { continue }; - if !self.project_dir.join(other).exists() { + if !cached_exists(&mut exists_cache, &self.project_dir, other) { continue; } - partners.push(c); + partners.push(c.clone()); } partners.sort_by(|a, b| { let na = a.get("co_commit_count").and_then(|v| v.as_i64()).unwrap_or(0); @@ -1901,11 +1921,11 @@ impl Engine { nb.cmp(&na) }); let mut neighbors = BTreeSet::new(); - for e in self.store.array("import_edges") { + for e in self.store.rows("import_edges") { let a = e.get("importer_file").and_then(|v| v.as_str()).unwrap_or(""); let b = e.get("imported_file").and_then(|v| v.as_str()).unwrap_or(""); - if a == file && b != file && self.project_dir.join(b).exists() { neighbors.insert(b.to_string()); } - if b == file && a != file && self.project_dir.join(a).exists() { neighbors.insert(a.to_string()); } + if a == file && b != file && cached_exists(&mut exists_cache, &self.project_dir, b) { neighbors.insert(b.to_string()); } + if b == file && a != file && cached_exists(&mut exists_cache, &self.project_dir, a) { neighbors.insert(a.to_string()); } } let untracked = self.working_tree && self.store.hash_for_file(file).is_empty(); if untracked && neighbors.is_empty() { @@ -1949,11 +1969,11 @@ impl Engine { let part = sub.suggest_directory(dir)?; if let Some(pairs) = part.as_object() { for (k, v) in pairs { - merged.insert(k.clone(), v.clone()); + merged.insert(k.to_string(), v.clone()); } } } - return Ok(Value::object(merged.into_iter().collect())); + return Ok(Value::object(merged.into_iter().map(|(k, v)| (crate::json::SharedStr::from(k), v)).collect())); } self.store.reload()?; if !self.has_data() { @@ -1967,7 +1987,7 @@ impl Engine { let suggested = self.suggest_tests(&file)?; if let Some(rows) = suggested.as_array() { if !rows.is_empty() { - pairs.push((file, Value::Array(rows.to_vec()))); + pairs.push((crate::json::SharedStr::from(file), Value::Array(rows.to_vec()))); } } } diff --git a/src/git.rs b/src/git.rs index 6c1970d..28841a4 100644 --- a/src/git.rs +++ b/src/git.rs @@ -101,6 +101,10 @@ impl GitAnalyzer { Ok(self.run(&["rev-parse", "--abbrev-ref", "HEAD"])?.trim().to_string()) } + pub fn head_sha(&self) -> Result { + Ok(self.run(&["rev-parse", "HEAD"])?.trim().to_string()) + } + pub fn branch_exists(&self, name: &str) -> bool { if name.starts_with('-') { return false; diff --git a/src/http.rs b/src/http.rs index 09d838a..cad7ae1 100644 --- a/src/http.rs +++ b/src/http.rs @@ -179,6 +179,7 @@ fn route(method: &str, path: &str, body: &[u8], engine: &Arc>) -> #[cfg(test)] mod tests { use super::*; + use crate::json::SharedStr; use std::time::{SystemTime, UNIX_EPOCH}; fn dummy_arg(tool: &str, field: &str) -> Value { @@ -195,11 +196,11 @@ mod tests { for field in crate::schemas::required_fields(tool) { if field == omit { if let Some(value) = bad.clone() { - pairs.push((field, value)); + pairs.push((SharedStr::from(field), value)); } continue; } - pairs.push((field.clone(), dummy_arg(tool, &field))); + pairs.push((SharedStr::from(field.clone()), dummy_arg(tool, &field))); } Value::object(pairs) } diff --git a/src/impact.rs b/src/impact.rs index 5b2e57c..e029fdd 100644 --- a/src/impact.rs +++ b/src/impact.rs @@ -1,6 +1,6 @@ //! Risk, churn, coupling, and impact ranking. Weights match risk_meta.py. -use crate::json::{py_round, quantize_gap, Value}; +use crate::json::{py_round, quantize_gap, SharedStr, Value}; use std::collections::{BTreeMap, BTreeSet}; pub const COCHANGE_CAP: f64 = 10.0; @@ -308,7 +308,7 @@ pub fn build_risk_meta(files: &[Value], commits: i64, test_edges: i64, test_resu let groups: Vec<_> = counts.into_iter().filter(|(_, c)| *c >= 3).collect(); if !groups.is_empty() { let max_group = groups.iter().map(|(_, c)| *c).max().unwrap_or(0); - meta.push(("uniform_risk_groups".into(), Value::object(groups.into_iter().map(|(k, c)| (k, Value::Int(c))).collect()))); + meta.push(("uniform_risk_groups".into(), Value::object(groups.into_iter().map(|(k, c)| (SharedStr::from(k), Value::Int(c))).collect()))); meta.push(("max_identical_risk_files".into(), Value::Int(max_group))); if max_group >= 5 { meta.push(("warnings".into(), Value::Array(vec![Value::String(format!( diff --git a/src/json.rs b/src/json.rs index 9541fb9..88b1765 100644 --- a/src/json.rs +++ b/src/json.rs @@ -3,6 +3,82 @@ use std::collections::BTreeMap; use std::fmt; +use std::ops::Deref; +use std::sync::Arc; + +/// A string that either owns its bytes or points into the parsed document. +/// Repeated object keys share the document buffer instead of allocating. +#[derive(Clone)] +pub struct SharedStr { + buf: Arc, + start: u32, + end: u32, +} + +impl SharedStr { + fn owned(s: &str) -> Self { + let buf: Arc = Arc::from(s); + let end = s.len() as u32; + Self { buf, start: 0, end } + } + + fn slice(buf: &Arc, start: usize, end: usize) -> Self { + Self { buf: Arc::clone(buf), start: start as u32, end: end as u32 } + } + + pub fn as_str(&self) -> &str { + &self.buf[self.start as usize..self.end as usize] + } +} + +impl Deref for SharedStr { + type Target = str; + fn deref(&self) -> &str { + self.as_str() + } +} + +impl PartialEq for SharedStr { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} +impl Eq for SharedStr {} +impl PartialOrd for SharedStr { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for SharedStr { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } +} +impl PartialEq for SharedStr { + fn eq(&self, other: &str) -> bool { + self.as_str() == other + } +} +impl fmt::Debug for SharedStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.as_str(), f) + } +} +impl fmt::Display for SharedStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} +impl From<&str> for SharedStr { + fn from(s: &str) -> Self { + Self::owned(s) + } +} +impl From for SharedStr { + fn from(s: String) -> Self { + Self::owned(&s) + } +} #[derive(Clone, Debug)] pub enum Value { @@ -11,8 +87,10 @@ pub enum Value { Int(i64), Float(f64), String(String), + /// Parsed text that shares the document allocation. + Shared(SharedStr), Array(Vec), - Object(Vec<(String, Value)>), + Object(Vec<(SharedStr, Value)>), } impl PartialEq for Value { @@ -21,10 +99,12 @@ impl PartialEq for Value { (Value::Null, Value::Null) => true, (Value::Bool(a), Value::Bool(b)) => a == b, (Value::String(a), Value::String(b)) => a == b, + (Value::Shared(a), Value::Shared(b)) => a == b, + (Value::String(a), Value::Shared(b)) | (Value::Shared(b), Value::String(a)) => a == b.as_str(), (Value::Array(a), Value::Array(b)) => a == b, (Value::Object(a), Value::Object(b)) => { - let ma = BTreeMap::from_iter(a.iter().map(|(k, v)| (k, v))); - let mb = BTreeMap::from_iter(b.iter().map(|(k, v)| (k, v))); + let ma = BTreeMap::from_iter(a.iter().map(|(k, v)| (k.as_str(), v))); + let mb = BTreeMap::from_iter(b.iter().map(|(k, v)| (k.as_str(), v))); ma == mb } (Value::Int(a), Value::Int(b)) => a == b, @@ -49,18 +129,18 @@ impl fmt::Display for Error { } impl Value { - pub fn object(pairs: Vec<(String, Value)>) -> Self { + pub fn object(pairs: Vec<(SharedStr, Value)>) -> Self { Value::Object(pairs) } - pub fn as_object(&self) -> Option<&[(String, Value)]> { + pub fn as_object(&self) -> Option<&[(SharedStr, Value)]> { match self { Value::Object(p) => Some(p), _ => None, } } - pub fn as_object_mut(&mut self) -> Option<&mut Vec<(String, Value)>> { + pub fn as_object_mut(&mut self) -> Option<&mut Vec<(SharedStr, Value)>> { match self { Value::Object(p) => Some(p), _ => None, @@ -80,7 +160,7 @@ impl Value { if let Some(slot) = self.get_mut(key) { *slot = value; } else if let Some(pairs) = self.as_object_mut() { - pairs.push((key.to_string(), value)); + pairs.push((SharedStr::owned(key), value)); } } @@ -101,6 +181,7 @@ impl Value { pub fn as_str(&self) -> Option<&str> { match self { Value::String(s) => Some(s), + Value::Shared(s) => Some(s.as_str()), _ => None, } } @@ -136,7 +217,7 @@ pub fn parse(input: &str) -> Result { return Err(Error { message: "json exceeds 32MiB".into() }); } let mut p = Parser { - bytes: input.as_bytes(), + buf: Arc::from(input), i: 0, depth: 0, }; @@ -152,19 +233,19 @@ pub fn parse(input: &str) -> Result { Ok(v) } -struct Parser<'a> { - bytes: &'a [u8], +struct Parser { + buf: Arc, i: usize, depth: u32, } -impl<'a> Parser<'a> { +impl Parser { fn eof(&self) -> bool { - self.i >= self.bytes.len() + self.i >= self.buf.len() } fn peek(&self) -> Option { - self.bytes.get(self.i).copied() + self.buf.as_bytes().get(self.i).copied() } fn bump(&mut self) -> Option { @@ -201,7 +282,7 @@ impl<'a> Parser<'a> { Some(b'n') => self.consume_ident(b"null").map(|_| Value::Null), Some(b't') => self.consume_ident(b"true").map(|_| Value::Bool(true)), Some(b'f') => self.consume_ident(b"false").map(|_| Value::Bool(false)), - Some(b'"') => self.parse_string().map(Value::String), + Some(b'"') => self.parse_string().map(Value::Shared), Some(b'[') => self.parse_array(), Some(b'{') => self.parse_object(), Some(b'-') | Some(b'0'..=b'9') => self.parse_number(), @@ -254,7 +335,7 @@ impl<'a> Parser<'a> { fn parse_object(&mut self) -> Result { self.enter()?; self.bump(); - let mut pairs: Vec<(String, Value)> = Vec::new(); + let mut pairs: Vec<(SharedStr, Value)> = Vec::new(); self.skip_ws(); if self.peek() == Some(b'}') { self.bump(); @@ -274,7 +355,7 @@ impl<'a> Parser<'a> { return Err(self.err("expected colon")); } let value = self.parse_value()?; - if let Some(slot) = pairs.iter_mut().rev().find(|(k, _)| k == &key) { + if let Some(slot) = pairs.iter_mut().rev().find(|(k, _)| k.as_str() == key.as_str()) { slot.1 = value; } else { pairs.push((key, value)); @@ -299,11 +380,36 @@ impl<'a> Parser<'a> { Ok(Value::Object(pairs)) } - fn parse_string(&mut self) -> Result { + fn parse_string(&mut self) -> Result { if self.bump() != Some(b'"') { return Err(self.err("expected string")); } + // Unescaped strings are a slice of the document. No per-string allocation. + let start = self.i; + let len = self.buf.len(); + let mut i = start; + while i < len { + let b = self.buf.as_bytes()[i]; + if b == b'"' { + let shared = SharedStr::slice(&self.buf, start, i); + self.i = i + 1; + return Ok(shared); + } + if b == b'\\' || b < 0x20 { + break; + } + i += 1; + } + self.i = i; let mut out = String::new(); + if i > start { + out.push_str(&self.buf[start..i]); + } + let owned = self.parse_string_tail(out)?; + Ok(SharedStr::owned(&owned)) + } + + fn parse_string_tail(&mut self, mut out: String) -> Result { loop { match self.bump() { None => return Err(self.err("unterminated string")), @@ -334,15 +440,12 @@ impl<'a> Parser<'a> { } Some(b) if b < 0x20 => return Err(self.err("raw control character in string")), Some(b) => { - // Decode one UTF-8 scalar starting at the byte we already consumed. let start = self.i - 1; let width = utf8_width(b).ok_or_else(|| self.err("bad utf-8"))?; - if start + width > self.bytes.len() { + if start + width > self.buf.len() { return Err(self.err("bad utf-8")); } - let slice = &self.bytes[start..start + width]; - let s = std::str::from_utf8(slice).map_err(|_| self.err("bad utf-8"))?; - out.push_str(s); + out.push_str(&self.buf[start..start + width]); self.i = start + width; } } @@ -389,7 +492,7 @@ impl<'a> Parser<'a> { self.bump(); } } - let text = std::str::from_utf8(&self.bytes[start..self.i]).unwrap(); + let text = self.buf[start..self.i].to_string(); if !is_float { if let Ok(i) = text.parse::() { return Ok(Value::Int(i)); @@ -442,6 +545,7 @@ fn write_value(out: &mut String, value: &Value, indent: usize, pretty: bool) -> out.push_str(&format_float(*f)); } Value::String(s) => write_string(out, s), + Value::Shared(s) => write_string(out, s.as_str()), Value::Array(items) => { out.push('['); if items.is_empty() { @@ -475,7 +579,7 @@ fn write_value(out: &mut String, value: &Value, indent: usize, pretty: bool) -> out.push('\n'); push_indent(out, indent + 1); } - write_string(out, k); + write_string(out, k.as_str()); out.push(':'); if pretty { out.push(' '); @@ -503,18 +607,33 @@ fn push_indent(out: &mut String, n: usize) { fn write_string(out: &mut String, s: &str) { out.push('"'); - for ch in s.chars() { - match ch { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - c if (c as u32) < 0x20 => { - out.push_str(&format!("\\u{:04x}", c as u32)); + let bytes = s.as_bytes(); + let mut start = 0; + for (i, &b) in bytes.iter().enumerate() { + let esc: &str = match b { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\n' => "\\n", + b'\r' => "\\r", + b'\t' => "\\t", + c if c < 0x20 => { + if start < i { + out.push_str(&s[start..i]); + } + out.push_str(&format!("\\u{c:04x}")); + start = i + 1; + continue; } - c => out.push(c), + _ => continue, + }; + if start < i { + out.push_str(&s[start..i]); } + out.push_str(esc); + start = i + 1; + } + if start < s.len() { + out.push_str(&s[start..]); } out.push('"'); } @@ -560,8 +679,10 @@ mod tests { #[test] fn roundtrip_and_duplicate_keys() { - let v = parse(r#"{"a":1,"b":[true,null,"x"],"a":2}"#).unwrap(); + let v = parse(r#"{"a":1,"b":[true,null,"x\"y"],"a":2,"u":"héllo\n"}"#).unwrap(); assert_eq!(v.get("a").and_then(|x| x.as_i64()), Some(2)); + assert_eq!(v.get("b").and_then(|x| x.as_array()).and_then(|a| a.get(2)).and_then(|x| x.as_str()), Some("x\"y")); + assert_eq!(v.get("u").and_then(|x| x.as_str()), Some("héllo\n")); let s = to_string(&v).unwrap(); let again = parse(&s).unwrap(); assert_eq!(v, again); diff --git a/src/store.rs b/src/store.rs index 36256e3..d05c1d0 100644 --- a/src/store.rs +++ b/src/store.rs @@ -34,13 +34,15 @@ pub struct Store { loaded_generation: i64, /// Test hook: the next N saves fail as a stale generation and do not publish. pub fail_stale_remaining: i64, - units_by_id: HashMap, - units_by_file: HashMap>, - edges_by_code: HashMap>, - churn_by_file: HashMap>, + /// `(len, mtime_nanos)` of the snapshot last parsed. A matching stamp skips a re-parse. + snapshot_stamp: Option<(u64, u128)>, + units_by_id: HashMap, + units_by_file: HashMap>, + edges_by_code: HashMap>, + churn_by_file: HashMap>, hash_by_file: HashMap, - blame_by_file: HashMap>, - co_by_file: HashMap>, + blame_by_file: HashMap>, + co_by_file: HashMap>, } impl Store { @@ -64,6 +66,7 @@ impl Store { doc: empty_doc(), loaded_generation: 0, fail_stale_remaining: 0, + snapshot_stamp: None, units_by_id: HashMap::new(), units_by_file: HashMap::new(), edges_by_code: HashMap::new(), @@ -92,8 +95,14 @@ impl Store { pub fn reload(&mut self) -> Result<(), StoreError> { { let _guard = self.lock.shared().map_err(|e| StoreError(e.to_string()))?; - self.doc = read_snapshot(&self.snapshot_path())?; + let path = self.snapshot_path(); + let stamp = file_stamp(&path); + if stamp.is_some() && self.snapshot_stamp == stamp { + return Ok(()); + } + self.doc = read_snapshot(&path)?; self.loaded_generation = self.doc.get("generation").and_then(|v| v.as_i64()).unwrap_or(0); + self.snapshot_stamp = file_stamp(&path); } self.reindex(); Ok(()) @@ -125,6 +134,7 @@ impl Store { write_snapshot_file(&self.temp_path(), &self.doc)?; self.publish_temp()?; self.loaded_generation = gen; + self.snapshot_stamp = file_stamp(&self.snapshot_path()); } self.reindex(); Ok(()) @@ -167,39 +177,50 @@ impl Store { } pub fn array(&self, key: &str) -> Vec { - self.doc.get(key).and_then(|v| v.as_array()).unwrap_or(&[]).to_vec() + self.rows(key).to_vec() + } + + pub fn rows(&self, key: &str) -> &[Value] { + self.doc.get(key).and_then(|v| v.as_array()).unwrap_or(&[]) } pub fn array_len(&self, key: &str) -> usize { - self.doc.get(key).and_then(|v| v.as_array()).map(|a| a.len()).unwrap_or(0) + self.rows(key).len() } - pub fn units_by_file(&self, file: &str) -> Vec { - self.units_by_file.get(file).cloned().unwrap_or_default() + pub fn units_by_file<'a>(&'a self, file: &str) -> Vec<&'a Value> { + self.indexed("code_units", self.units_by_file.get(file)) } - pub fn unit_by_id(&self, id: &str) -> Option { - self.units_by_id.get(id).cloned() + pub fn unit_by_id<'a>(&'a self, id: &str) -> Option<&'a Value> { + let idx = *self.units_by_id.get(id)?; + self.rows("code_units").get(idx) } - pub fn edges_for_code(&self, code_id: &str) -> Vec { - self.edges_by_code.get(code_id).cloned().unwrap_or_default() + pub fn edges_for_code<'a>(&'a self, code_id: &str) -> Vec<&'a Value> { + self.indexed("test_edges", self.edges_by_code.get(code_id)) } - pub fn churn_for_file(&self, file: &str) -> Vec { - self.churn_by_file.get(file).cloned().unwrap_or_default() + pub fn churn_for_file<'a>(&'a self, file: &str) -> Vec<&'a Value> { + self.indexed("churn", self.churn_by_file.get(file)) } pub fn hash_for_file(&self, file: &str) -> String { self.hash_by_file.get(file).cloned().unwrap_or_default() } - pub fn blame_for_file(&self, file: &str) -> Vec { - self.blame_by_file.get(file).cloned().unwrap_or_default() + pub fn blame_for_file<'a>(&'a self, file: &str) -> Vec<&'a Value> { + self.indexed("blame", self.blame_by_file.get(file)) } - pub fn co_for_file(&self, file: &str) -> Vec { - self.co_by_file.get(file).cloned().unwrap_or_default() + pub fn co_for_file<'a>(&'a self, file: &str) -> Vec<&'a Value> { + self.indexed("co_changes", self.co_by_file.get(file)) + } + + fn indexed<'a>(&'a self, key: &str, idxs: Option<&Vec>) -> Vec<&'a Value> { + let Some(idxs) = idxs else { return Vec::new() }; + let rows = self.rows(key); + idxs.iter().filter_map(|i| rows.get(*i)).collect() } pub fn reindex_memory(&mut self) { @@ -207,12 +228,6 @@ impl Store { } fn reindex(&mut self) { - let units = self.doc.get("code_units").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); - let edges = self.doc.get("test_edges").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); - let churn = self.doc.get("churn").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); - let hashes = self.doc.get("file_hashes").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); - let blame = self.doc.get("blame").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); - let co = self.doc.get("co_changes").and_then(|v| v.as_array()).map(|a| a.to_vec()).unwrap_or_default(); self.units_by_id.clear(); self.units_by_file.clear(); self.edges_by_code.clear(); @@ -220,47 +235,66 @@ impl Store { self.hash_by_file.clear(); self.blame_by_file.clear(); self.co_by_file.clear(); - for row in units { - if let Some(id) = row.get("id").and_then(|v| v.as_str()) { - self.units_by_id.insert(id.to_string(), row.clone()); - } - if let Some(file) = row.get("file_path").and_then(|v| v.as_str()) { - self.units_by_file.entry(file.to_string()).or_default().push(row); - } + index_by(self.doc.get("code_units"), "id", "file_path", &mut self.units_by_id, &mut self.units_by_file); + index_one(self.doc.get("test_edges"), "code_id", &mut self.edges_by_code); + index_one(self.doc.get("churn"), "file_path", &mut self.churn_by_file); + index_hashes(self.doc.get("file_hashes"), &mut self.hash_by_file); + index_one(self.doc.get("blame"), "file_path", &mut self.blame_by_file); + index_co(self.doc.get("co_changes"), &mut self.co_by_file); + } +} + +fn index_by(value: Option<&Value>, id_key: &str, file_key: &str, by_id: &mut HashMap, by_file: &mut HashMap>) { + let Some(rows) = value.and_then(|v| v.as_array()) else { return }; + for (i, row) in rows.iter().enumerate() { + if let Some(id) = row.get(id_key).and_then(|v| v.as_str()) { + by_id.insert(id.to_string(), i); } - for row in edges { - if let Some(id) = row.get("code_id").and_then(|v| v.as_str()) { - self.edges_by_code.entry(id.to_string()).or_default().push(row); - } + if let Some(file) = row.get(file_key).and_then(|v| v.as_str()) { + by_file.entry(file.to_string()).or_default().push(i); } - for row in churn { - if let Some(file) = row.get("file_path").and_then(|v| v.as_str()) { - self.churn_by_file.entry(file.to_string()).or_default().push(row); - } + } +} + +fn index_one(value: Option<&Value>, key: &str, into: &mut HashMap>) { + let Some(rows) = value.and_then(|v| v.as_array()) else { return }; + for (i, row) in rows.iter().enumerate() { + if let Some(id) = row.get(key).and_then(|v| v.as_str()) { + into.entry(id.to_string()).or_default().push(i); } - for row in hashes { - if let (Some(file), Some(hash)) = (row.get("file_path").and_then(|v| v.as_str()), row.get("hash").and_then(|v| v.as_str())) { - self.hash_by_file.insert(file.to_string(), hash.to_string()); - } + } +} + +fn index_hashes(value: Option<&Value>, into: &mut HashMap) { + let Some(rows) = value.and_then(|v| v.as_array()) else { return }; + for row in rows { + if let (Some(file), Some(hash)) = (row.get("file_path").and_then(|v| v.as_str()), row.get("hash").and_then(|v| v.as_str())) { + into.insert(file.to_string(), hash.to_string()); } - for row in blame { - if let Some(file) = row.get("file_path").and_then(|v| v.as_str()) { - self.blame_by_file.entry(file.to_string()).or_default().push(row); - } + } +} + +fn index_co(value: Option<&Value>, into: &mut HashMap>) { + let Some(rows) = value.and_then(|v| v.as_array()) else { return }; + for (i, row) in rows.iter().enumerate() { + let a = row.get("file_a").and_then(|v| v.as_str()).unwrap_or(""); + let b = row.get("file_b").and_then(|v| v.as_str()).unwrap_or(""); + if !a.is_empty() { + into.entry(a.to_string()).or_default().push(i); } - for row in co { - let a = row.get("file_a").and_then(|v| v.as_str()).unwrap_or(""); - let b = row.get("file_b").and_then(|v| v.as_str()).unwrap_or(""); - if !a.is_empty() { - self.co_by_file.entry(a.to_string()).or_default().push(row.clone()); - } - if !b.is_empty() && b != a { - self.co_by_file.entry(b.to_string()).or_default().push(row); - } + if !b.is_empty() && b != a { + into.entry(b.to_string()).or_default().push(i); } } } +fn file_stamp(path: &Path) -> Option<(u64, u128)> { + let meta = fs::metadata(path).ok()?; + let modified = meta.modified().ok()?; + let nanos = modified.duration_since(std::time::UNIX_EPOCH).ok()?.as_nanos(); + Some((meta.len(), nanos)) +} + pub fn reject_storage_dir(dir: &Path) -> Result<(), StoreError> { let raw = dir.to_string_lossy(); if raw == ":memory:" || raw.starts_with("file:") { diff --git a/src/syntax.rs b/src/syntax.rs index 1d42381..0b736eb 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -50,32 +50,35 @@ fn parse_python(source: &str) -> Node { let lines: Vec<&str> = source.split('\n').map(|l| l.trim_end_matches('\r')).collect(); let mut module = Node { kind: "module", name: String::new(), line_start: 1, line_end: lines.len().max(1), children: Vec::new() }; let mut depth = 0i32; - let mut stack: Vec<(usize, usize, String)> = Vec::new(); + let mut quote: Quote = Quote::None; + // (indent, is_class, bare class name). Functions are pushed so a nested def + // is not qualified; only a def whose enclosing block is a class becomes Class.method. + let mut stack: Vec<(usize, bool, String)> = Vec::new(); let mut i = 0; while i < lines.len() { - let (clean, next_depth) = strip_line(lines[i], depth); + let (clean, next_depth, next_quote) = scan_py_line(lines[i], depth, quote); let trimmed = clean.trim(); - if depth == 0 { + let raw_trim = lines[i].trim(); + let skip_stack = raw_trim.is_empty() || raw_trim.starts_with('#') || quote != Quote::None; + if depth == 0 && quote == Quote::None && !skip_stack { let indent = indent_of(lines[i]); - stack.retain(|(_, ind, _)| *ind < indent); + stack.retain(|(ind, _, _)| *ind < indent); if let Some(name) = class_name(trimmed) { let end = block_end(&lines, i, indent); - let parent = stack.iter().rev().find(|(_, ind, _)| indent > *ind).map(|(_, _, n)| n.clone()); - let qname = match parent { - Some(p) => format!("{p}.{name}"), - None => name, - }; module.children.push(Node { kind: "class", - name: qname.clone(), + name: name.clone(), line_start: i + 1, line_end: end, children: Vec::new(), }); - stack.push((module.children.len() - 1, indent, qname)); + stack.push((indent, true, name)); } else if let Some((name, is_async)) = def_name(trimmed) { - let parent = stack.iter().rev().find(|(_, ind, _)| indent > *ind).map(|(_, _, n)| n.clone()); - let qname = match parent { + let parent = match stack.last() { + Some((_, true, class_name)) => Some(class_name.clone()), + _ => None, + }; + let qname = match &parent { Some(p) => format!("{p}.{name}"), None => name, }; @@ -87,9 +90,11 @@ fn parse_python(source: &str) -> Node { line_end: end, children: Vec::new(), }); + stack.push((indent, false, String::new())); } } depth = next_depth; + quote = next_quote; i += 1; } module @@ -244,34 +249,114 @@ fn bracket_delta(line: &str) -> i32 { d } -fn strip_line(line: &str, depth_in: i32) -> (String, i32) { +#[derive(Clone, Copy, PartialEq, Eq)] +enum Quote { + None, + Single(char), + Triple(char), +} + +/// Track parentheses and quotes, including triple-quoted strings that span lines. +/// Parentheses inside strings do not hide the next `def`. +fn scan_py_line(line: &str, depth_in: i32, mut quote: Quote) -> (String, i32, Quote) { let mut out = String::new(); let mut depth = depth_in; - let mut chars = line.chars().peekable(); - let mut in_str: Option = None; - while let Some(c) = chars.next() { - if let Some(q) = in_str { - if c == '\\' { chars.next(); continue; } - if c == q { in_str = None; } + let bytes = line.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let c = bytes[i] as char; + if let Quote::Triple(q) = quote { + if bytes[i..].starts_with(&[q as u8, q as u8, q as u8]) { + quote = Quote::None; + i += 3; + continue; + } + i += 1; continue; } - if depth > 0 { - match c { - '"' | '\'' => in_str = Some(c), - '(' | '[' | '{' => depth += 1, - ')' | ']' | '}' => depth -= 1, - _ => {} + if let Quote::Single(q) = quote { + if c == '\\' { + i += 1; + if i < bytes.len() { i += 1; } + continue; + } + if c == q { + quote = Quote::None; } + i += 1; + continue; + } + if c == '#' && depth == 0 { + break; + } + if c == '"' || c == '\'' { + if i + 2 < bytes.len() && bytes[i + 1] == bytes[i] && bytes[i + 2] == bytes[i] { + quote = Quote::Triple(c); + i += 3; + continue; + } + quote = Quote::Single(c); + i += 1; continue; } - if c == '#' { break; } - if c == '"' || c == '\'' { in_str = Some(c); continue; } match c { - '(' | '[' | '{' => { depth += 1; out.push(c); } - ')' | ']' | '}' => { depth -= 1; out.push(c); } - _ => out.push(c), + '(' | '[' | '{' => { + depth += 1; + if depth_in == 0 || depth == 1 { + out.push(c); + } + } + ')' | ']' | '}' => { + depth -= 1; + out.push(c); + } + _ => { + if depth <= 0 { + out.push(c); + } + } } + i += 1; + } + (out, depth.max(0), quote) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn triple_quoted_parens_do_not_hide_later_defs() { + let src = r#" +class MigrationTests: + def test_create_model(self): + sql = """ + CREATE TABLE x (id int) + """ + return 1 + + def test_create_model2(self): + return 2 + + def test_outer(self): + def test_inner(): + return 3 + +def test_after(): + pass +"#; + let units = python_units("t.py", src); + let names: Vec<&str> = units.iter().map(|u| u.name.as_str()).collect(); + assert_eq!( + names, + vec![ + "MigrationTests", + "MigrationTests.test_create_model", + "MigrationTests.test_create_model2", + "MigrationTests.test_outer", + "test_inner", + "test_after", + ] + ); } - let _ = depth_in; - (out, depth.max(0)) }