From c63c737f1121af100b12560e839d0b8366d3fa73 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 21 Sep 2026 18:52:20 +1000 Subject: [PATCH 1/2] Adopt inspect-evals-lint 0.4.2 in place of the vendored autolint (template v1.2.0) The lint-related managed files are merged from the template (checks.yml and run_checks.sh run `uv run inspect-evals-lint --all`, AUTOMATED_CHECKS.md documents the rules and the new suppression syntax); tools/run_autolint is removed; pyproject.toml gains the dev dependency and the [tool.inspect-evals-lint] table. The lab_bench_2 extra declares pillow and pydantic, which the evaluation imports directly. Co-Authored-By: Claude Fable 5.1 --- .claude/skills/create-eval/SKILL.md | 2 +- .claude/skills/ensure-test-coverage/SKILL.md | 6 +- .github/workflows/checks.yml | 6 +- .github/workflows/sync-template.yml | 6 +- .template-sync-sha | 2 +- .template-version | 2 +- AGENTS.md | 2 +- AUTOMATED_CHECKS.md | 32 +- CONTRIBUTING.md | 26 +- MANAGED_FILES.md | 14 +- pyproject.toml | 14 +- tools/run_autolint.py | 146 -------- tools/run_autolint/__init__.py | 279 -------------- tools/run_autolint/checks/__init__.py | 59 --- tools/run_autolint/checks/best_practices.py | 323 ---------------- tools/run_autolint/checks/code_quality.py | 98 ----- tools/run_autolint/checks/dependencies.py | 308 ---------------- tools/run_autolint/checks/file_structure.py | 360 ------------------ tools/run_autolint/checks/tests.py | 369 ------------------- tools/run_autolint/checks/utils.py | 164 --------- tools/run_autolint/models.py | 89 ----- tools/run_autolint/output.py | 245 ------------ tools/run_autolint/suppressions.py | 77 ---- tools/run_checks.sh | 2 +- uv.lock | 19 + 25 files changed, 91 insertions(+), 2559 deletions(-) delete mode 100755 tools/run_autolint.py delete mode 100644 tools/run_autolint/__init__.py delete mode 100644 tools/run_autolint/checks/__init__.py delete mode 100644 tools/run_autolint/checks/best_practices.py delete mode 100644 tools/run_autolint/checks/code_quality.py delete mode 100644 tools/run_autolint/checks/dependencies.py delete mode 100644 tools/run_autolint/checks/file_structure.py delete mode 100644 tools/run_autolint/checks/tests.py delete mode 100644 tools/run_autolint/checks/utils.py delete mode 100644 tools/run_autolint/models.py delete mode 100644 tools/run_autolint/output.py delete mode 100644 tools/run_autolint/suppressions.py diff --git a/.claude/skills/create-eval/SKILL.md b/.claude/skills/create-eval/SKILL.md index bf48e8a..11e42c9 100644 --- a/.claude/skills/create-eval/SKILL.md +++ b/.claude/skills/create-eval/SKILL.md @@ -461,7 +461,7 @@ Fix any failures. For tests requiring Docker or datasets, note which tests need ### 6.1 Run Autolint ```bash -uv run python tools/run_autolint.py +uv run inspect-evals-lint ``` The autolint surfaces the registry's recommended structural standards. In diff --git a/.claude/skills/ensure-test-coverage/SKILL.md b/.claude/skills/ensure-test-coverage/SKILL.md index b3ae8d5..ba0ca8e 100644 --- a/.claude/skills/ensure-test-coverage/SKILL.md +++ b/.claude/skills/ensure-test-coverage/SKILL.md @@ -71,7 +71,7 @@ Check if the eval has multiple `@task` functions or a single task with parameter Run all autolint checks in one command to get a baseline of structural test coverage: ```bash -uv run python tools/run_autolint.py +uv run inspect-evals-lint ``` This checks: `tests_exist`, `tests_init`, `e2e_test`, `record_to_sample_test`, `custom_solver_tests`, `custom_scorer_tests`, `custom_tool_tests` (plus non-test checks). You do not need to report to the user specific details of this beyond what the autolint tool itself tells them. @@ -307,7 +307,7 @@ uv run ruff check tests// uv run ruff format tests// # Re-run autolint to verify all checks pass -uv run python tools/run_autolint.py +uv run inspect-evals-lint # Re-measure coverage uv run pytest tests// --cov=src/ --cov-report=term-missing -q --runslow @@ -320,7 +320,7 @@ Fix any failures before presenting results. Show the user a summary: 1. **Component inventory**: What testable components were found -2. **Autolint results**: Pass/fail for each check (from `run_autolint.py` output) +2. **Autolint results**: Pass/fail for each check (from `inspect-evals-lint` output) 3. **Coverage**: Overall and per-file percentages, with missed line analysis 4. **Quality gaps**: Issues found in Phase 2.3 5. **Actions taken**: If tests were created/modified, list what was done diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2241c67..370e7b5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -120,7 +120,7 @@ jobs: - uses: hynek/build-and-inspect-python-package@v2 autolint: - name: Autolint (inspect_evals standards) + name: Autolint (inspect-evals-lint) runs-on: ubuntu-latest needs: config # Default: advisory. Set ENFORCE_AUTOLINT=true in tools/enforcement.config @@ -134,8 +134,8 @@ jobs: python-version: "3.12" - name: Install dependencies run: uv sync --frozen - - name: Run autolint - run: uv run python tools/run_autolint.py --all-evals + - name: Run inspect-evals-lint + run: uv run inspect-evals-lint --all --output-format github generated-docs: name: Generated READMEs up to date diff --git a/.github/workflows/sync-template.yml b/.github/workflows/sync-template.yml index fa7f224..6c9043d 100644 --- a/.github/workflows/sync-template.yml +++ b/.github/workflows/sync-template.yml @@ -13,7 +13,7 @@ permissions: jobs: check-activity: runs-on: ubuntu-latest - if: github.repository != 'ArcadiaImpact/inspect-evals-template' + if: github.repository != 'Generality-Labs/inspect-evals-template' outputs: is_active: ${{ steps.activity.outputs.is_active }} steps: @@ -54,7 +54,7 @@ jobs: - name: Add template remote run: | - git remote add template https://github.com/ArcadiaImpact/inspect-evals-template.git 2>/dev/null || true + git remote add template https://github.com/Generality-Labs/inspect-evals-template.git 2>/dev/null || true git fetch template main - name: Extract managed files list @@ -154,7 +154,7 @@ jobs: # Build PR body body="## Template Sync" - body="$body\n\nThis PR updates managed files from the [inspect-eval-template](https://github.com/ArcadiaImpact/inspect-eval-template)." + body="$body\n\nThis PR updates managed files from the [inspect-evals-template](https://github.com/Generality-Labs/inspect-evals-template)." body="$body\n\n**Template version:** $template_version" if [ -n "$conflicts" ]; then diff --git a/.template-sync-sha b/.template-sync-sha index 335899a..713ba55 100644 --- a/.template-sync-sha +++ b/.template-sync-sha @@ -1 +1 @@ -aa74b30159bea953e238d4021315e24e4d60ad38 +d2caec89178e122a6f6aa0872a880c7c2920e419 diff --git a/.template-version b/.template-version index 795460f..79127d8 100644 --- a/.template-version +++ b/.template-version @@ -1 +1 @@ -v1.1.0 +v1.2.0 diff --git a/AGENTS.md b/AGENTS.md index 0bac1dc..20a9177 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,7 @@ This workflow runs a series of workflows each in turn. Each workflow is to be ru ### Useful Commands -1. You can see our linting in the `.github/workflows/checks.yml` file. Run `make check` (which delegates to `tools/run_checks.sh`) when checking linting locally — it runs ruff, mypy, autolint, and the rest in one go and reports advisory vs enforced failures. +1. You can see our linting in the `.github/workflows/checks.yml` file. Run `make check` (which delegates to `tools/run_checks.sh`) when checking linting locally — it runs ruff, mypy, `inspect-evals-lint`, and the rest in one go and reports advisory vs enforced failures. 2. To run tests, run `uv run pytest tests/`. 3. To run evaluations, run `uv run inspect eval /` (the eval name comes from the entry-point registered in `pyproject.toml`). 4. To run a specific task (i.e, a function with the @task decorator), run `uv run inspect eval /`. diff --git a/AUTOMATED_CHECKS.md b/AUTOMATED_CHECKS.md index bdcaabe..e28b63c 100644 --- a/AUTOMATED_CHECKS.md +++ b/AUTOMATED_CHECKS.md @@ -7,24 +7,30 @@ > are **recommended, not required** — autolint is advisory by default. See > [Checks and enforcement](README.md#checks-and-enforcement) to opt in. -We have a number of automatic checks that can be run using [tools/run_autolint.py](tools/run_autolint.py): +The automated checks are provided by the [inspect-evals-lint](https://github.com/Generality-Labs/inspect-evals-lint) package. It is installed through the `dev` dependency group and configured under `[tool.inspect-evals-lint]` in `pyproject.toml` (the `template` layout preset). Run it with: ```bash -uv run python tools/run_autolint.py +uv run inspect-evals-lint +uv run inspect-evals-lint --all # every evaluation and src/utils +uv run inspect-evals-lint --all --select IEBP # one category, or one rule by code or name +uv run inspect-evals-lint --explain IEBP002 # a rule's documentation ``` -To suppress a check, use: +Every check is a *rule* with a code (`IEFS` file structure, `IECQ` code quality, `IETS` tests, `IEBP` best practices, then a number) and a name; either works in `--select`, `--ignore` and suppression comments. A rule reports one finding per site with a file and line, so CI annotates pull requests at the offending lines. -- Line-level: `# noautolint: ` -- File-level: `# noautolint-file: ` (at top of file) -- Directory-level: Add check name to `src///.noautolint` -- Eval-level: Add check name to `src//.noautolint` +Evaluations live directly under `src/`, one package each, registered under `[project.entry-points.inspect_ai]`. `src/utils` is linted as a helper package: it runs the checks about code behaviour (private imports, score values, model roles, dependencies, tests for custom components) and not the ones about an evaluation's structure. `src/examples` is not linted. The package version is pinned in `pyproject.toml` and `uv.lock`; check implementations and their unit tests are maintained in the package repository. -You don't need to read these checks - they are presented here as a reference in case of linting errors. +To suppress a finding: + +- Line-level: `# inspect-evals-lint: ignore[]` on the offending line (or any line of a multi-line statement), with a rule name or code, several comma-separated. A bare `ignore` is a configuration error. +- File-level: `# inspect-evals-lint: ignore-file[]` within the first ten lines of the file. +- Directory or package: `per-file-ignores = { "src///**" = [""] }` under `[tool.inspect-evals-lint]` in `pyproject.toml`; `exclude` lists globs the code rules never read at all, for code shipped into a sandbox. + +You don't need to read these checks - they are presented here as a reference in case of linting errors. The canonical description of each rule, with the reasoning behind it, is its page at [inspect-evals-lint.generality.org](https://inspect-evals-lint.generality.org/CHECKS/); `--explain ` prints the same text. ## File Structure (Automated) -- The evaluation is located in a sub-directory of `src/` (`eval_location`) +- The evaluation is located in a sub-directory of `src/` (`package_location`) - `__init__.py` exports task and related functions (`init_exports`) - @task functions are contained within `src//.py` (`main_file`) - Task registered in `pyproject.toml` under `[project.entry-points.inspect_ai]` (`registry`) @@ -35,7 +41,9 @@ You don't need to read these checks - they are presented here as a reference in - No imports from private inspect_ai modules (those starting with `_`) (`private_api_imports`) - Score() calls use CORRECT/INCORRECT constants instead of literal strings (`score_constants`) -- External eval-specific dependencies declared in `pyproject.toml` (`external_dependencies`) +- `Score.unscored()` calls pass a `reason=`, and the former `metadata["unscored_reason"]` key does not appear (`unscored_reason`) +- Every suppression marker is one the linter reads (`suppression_syntax`, warns): a `# noautolint` comment or `.noautolint` file, a bare `ignore`, an `ignore-file` past the header or a selector naming no rule suppresses nothing, so it is reported at its line with the replacement in the hint. +- External eval-specific dependencies declared in `pyproject.toml` (`external_dependencies`). For `src/utils`, imports at module level must be in `[project].dependencies`, because every evaluation that imports the helper loads them; imports inside a function only need declaring in some optional group. ## Tests (Automated) @@ -50,6 +58,10 @@ You don't need to read these checks - they are presented here as a reference in ## Best Practices (Automated) - `get_model()` only called inside @solver/@scorer decorated functions (`get_model_location`) +- Model roles supply a model, a `default=`, or `required=True` to prevent an unbound role from falling back to the model under evaluation (`model_role_resolution`) - Sample() calls include an `id=` parameter for stable IDs (`sample_ids`) - @task functions provide defaults for overridable parameters (solver, scorer, etc.) (`task_overridable_defaults`) +- Sandbox images pulled from a registry in compose files use an immutable tag or `@sha256` digest (`sandbox_image_pinning`). Untagged and `:latest` references fail; services built locally via `build:` and references interpolated from environment variables are skipped. Exceptions go under `[tool.inspect-evals-lint.allowlists.sandbox_image_pinning]` in `pyproject.toml`. +- An evaluation whose `eval.yaml` declares `metadata.requires.gpu` ships a `_sandbox_check` task with `kind: maintenance` that certifies its image on GPU hardware (`gpu_sandbox_check`). +- Dockerfile builds consume locked inputs (`dockerfile_locking`, warns): dependency installs use a committed `uv.lock` (`uv sync --locked`) or a hashed snapshot (`pip install --require-hashes -r`), `FROM` and `COPY --from` images carry an `@sha256` digest, Git dependencies name a full commit, and nothing is piped from `curl` into a shell. OS package installs cannot be locked and are named in the pass message. - Dataset pinning is enforced at runtime: `hf_dataset()`, `load_dataset()`, `snapshot_download()`, and `hf_hub_download()` wrappers require a `revision=` keyword argument diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 324ef96..9829588 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -282,7 +282,7 @@ RUN_DATASET_DOWNLOAD_TESTS=1 #### CI workflows -- The template ships `.github/workflows/checks.yml` which runs ruff, mypy, the POSIX-code check, the unlisted-evals check, the package build, autolint, and a few advisory checks. By default this does not run pytest — the template assumes you run tests locally during development. If you want CI to run your tests, add a job to `checks.yml` (or a separate workflow) that calls `make test`. +- The template ships `.github/workflows/checks.yml` which runs ruff, mypy, the POSIX-code check, the unlisted-evals check, the package build, `inspect-evals-lint`, and a few advisory checks. By default this does not run pytest — the template assumes you run tests locally during development. If you want CI to run your tests, add a job to `checks.yml` (or a separate workflow) that calls `make test`. - The upstream `inspect_evals` registry has additional CI (a `build.yml` that runs the test suite with `RUN_SLOW_TESTS=no`, plus a nightly heavy-tests workflow that detects unmarked slow/docker tests). If your fork wants the same coverage, those workflows are good references but they aren't shipped here. ### Manual testing @@ -441,7 +441,7 @@ Each evaluation has an `eval.yaml` file in its directory (e.g., `src/ - `metadata`: (Optional) Object containing system/configuration information. All fields are optional. May include: - `sandbox`: List of components that use a sandboxed environment. Can include `"solver"` and/or `"scorer"`. Omit if no sandbox is used. - - `requires_internet`: Set to `true` only if the evaluation requires internet access. Omit if false. + - `requires`: Capabilities the evaluation needs from its environment. `internet: true` if the runner needs outbound network access beyond fetching packages. `gpu: true` if a GPU is needed where the sandbox runs, or a mapping with `count` and an informational `products` list (e.g. `[T4, H100]`) naming the hardware the eval is designed or verified for. Unknown capability keys are rejected. Omit if neither applies. - `environment`: Special environment requirements (e.g., `"Kali-Linux"`). Only include if a non-standard environment is needed. Example: @@ -449,12 +449,28 @@ Each evaluation has an `eval.yaml` file in its directory (e.g., `src/ ```yaml metadata: sandbox: ["solver", "scorer"] # Both solver and scorer use sandbox - requires_internet: true + requires: + internet: true + gpu: + count: 1 + products: [T4] environment: "Kali-Linux" # Only include if special environment is needed ``` Omit the entire metadata field if none of its subfields are needed. +- `kind` (per task): Set `kind: maintenance` on a task whose scores describe the evaluation harness rather than the model, such as a judge meta-evaluation or a sandbox image check. The default, `benchmark`, is what listings and evaluation reports assume; maintenance tasks are still registered and runnable but are listed separately in generated documentation. + + ```yaml + tasks: + - name: my_bench + dataset_samples: 500 + - name: my_bench_judge_check + dataset_samples: 200 + kind: maintenance + comment: Scores the judge against human labels; not a model result. + ``` + - `human_baseline`: Optional field for evaluations with known human performance metrics ```yaml @@ -483,7 +499,9 @@ dependency: "your_evaluation" # Optional tags: ["Agent"] # Optional metadata: # optional metadata documenting eval information. All fields optional sandbox: [] # list of eval aspects that use a sandbox, can include "solver" and/or "scorer" - requires_internet: true # boolean indicating whether the eval requires internet access + requires: # capabilities the eval needs; omit keys that do not apply + internet: true # runner needs outbound network access + gpu: false # or `true`, or {count: 1, products: [T4]} environment: "Kali-Linux" # optional environment information external_assets: # required; use [] if none - type: huggingface diff --git a/MANAGED_FILES.md b/MANAGED_FILES.md index d8b8e78..cbf13dc 100644 --- a/MANAGED_FILES.md +++ b/MANAGED_FILES.md @@ -65,18 +65,6 @@ This means that you can add or remove files to this section as needed. - `tools/generate_readmes.py` - `tools/inject_managed_prefix.sh` - `tools/list_large_files.sh` -- `tools/run_autolint.py` -- `tools/run_autolint/__init__.py` -- `tools/run_autolint/checks/__init__.py` -- `tools/run_autolint/checks/best_practices.py` -- `tools/run_autolint/checks/code_quality.py` -- `tools/run_autolint/checks/dependencies.py` -- `tools/run_autolint/checks/file_structure.py` -- `tools/run_autolint/checks/tests.py` -- `tools/run_autolint/checks/utils.py` -- `tools/run_autolint/models.py` -- `tools/run_autolint/output.py` -- `tools/run_autolint/suppressions.py` - `tools/run_checks.sh` - `tools/sync_merge_file.sh` @@ -90,7 +78,7 @@ These are yours — the sync mechanism will never modify them: - `tests//` — your tests - `tests/conftest.py` — pytest fixtures and markers; the upstream version depends on `inspect_evals` package internals and is not synced - `README.md` — your evaluation's documentation -- `pyproject.toml` — your project configuration (tool configs are managed, but `[project]` and `[project.entry-points]` sections are yours) +- `pyproject.toml` — your project configuration (tool configs are managed, but `[project]` and `[project.entry-points]` sections are yours). Keep the `inspect-evals-lint` entry in the `dev` dependency group and the `[tool.inspect-evals-lint]` table: `checks.yml` and `tools/run_checks.sh` run `uv run inspect-evals-lint`, which needs the package installed. Repositories created before template v1.2.0 add both by hand. - `agent_artefacts/` — runtime artefacts from agent workflows - `LICENSE` — your chosen license. By default, the MIT license will be copied over, but this can be altered by you at any time. diff --git a/pyproject.toml b/pyproject.toml index 00248a2..e77c417 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,13 @@ where = ["src"] [tool.setuptools.package-data] "*" = ["eval.yaml"] +# Evaluation checks (see AUTOMATED_CHECKS.md). The template preset expects one +# package per evaluation directly under src/, registered under +# [project.entry-points.inspect_ai]; src/utils is linted as a helper package +# and src/examples is not linted. +[tool.inspect-evals-lint] +preset = "template" + [tool.ruff] extend-exclude = [ "agent_artefacts", @@ -160,11 +167,16 @@ lab_bench_2 = "lab_bench_2" # Reference implementation's deterministic scoring code (CC BY-SA 4.0), scoped to # this eval rather than core. Pulled as a dependency rather than vendored; source # + pin in [tool.uv.sources]. See src/lab_bench_2/README.md. -lab_bench_2 = ["labbench2"] +lab_bench_2 = [ + "labbench2", + "pillow", # attachment_builder downscales oversized image attachments + "pydantic", # scorers validate the grader's structured output +] test = [] [dependency-groups] dev = [ + "inspect-evals-lint==0.4.2", # Evaluation checks; see AUTOMATED_CHECKS.md "anthropic>=0.105.0", "inspect-scout>=0.3.0", # llm_scanner API used by agent_artefacts/trajectory_analysis/inspect_scout/ "mypy", diff --git a/tools/run_autolint.py b/tools/run_autolint.py deleted file mode 100755 index b63ad3e..0000000 --- a/tools/run_autolint.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -""" -CLI entry point for autolint. - -Lint an evaluation against the AUTOMATED_CHECKS.md requirements. - -Usage: - python tools/run_autolint.py - python tools/run_autolint.py gpqa - python tools/run_autolint.py --all-evals # Check all evaluations - python tools/run_autolint.py gpqa --check registry # Run only one check - python tools/run_autolint.py --list-checks # List all available checks - python tools/run_autolint.py --check-summary # Per-check compliance across all evals - -Suppression: - Line-level: # noautolint: - File-level: # noautolint-file: (at top of file) - Eval-level: Create .noautolint file in eval directory with check names (one per line) - -Exit codes: - 0 - All checks passed - 1 - Some checks failed - 2 - Evaluation not found -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -# Add tools directory to path for imports -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from tools.run_autolint import ( # type: ignore[attr-defined] - get_all_check_names, - get_all_eval_names, - lint_evaluation, - resolve_repo_root, -) -from tools.run_autolint.output import ( - print_check_summary, - print_overall_summary, - print_report, -) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Lint an evaluation against AUTOMATED_CHECKS.md requirements" - ) - parser.add_argument( - "eval_name", - nargs="?", - help="Name of the evaluation to lint (e.g., 'gpqa')", - ) - parser.add_argument( - "--all-evals", - action="store_true", - help="Lint all evaluations", - ) - parser.add_argument( - "--check", - type=str, - default=None, - metavar="CHECK_NAME", - help="Run only a specific check (use --list-checks to see available checks)", - ) - parser.add_argument( - "--list-checks", - action="store_true", - help="List all available checks and exit", - ) - parser.add_argument( - "--summary-only", - action="store_true", - help="Only print summary (useful with --all-evals)", - ) - parser.add_argument( - "--check-summary", - action="store_true", - help="Show per-check compliance summary across all evals (implies --all-evals --summary-only)", - ) - args = parser.parse_args() - - # Handle --list-checks - if args.list_checks: - print("Available checks:") - for check_name in get_all_check_names(): - print(f" {check_name}") - sys.exit(0) - - # Validate --check argument early - if args.check: - all_checks = get_all_check_names() - if args.check not in all_checks: - print(f"Error: Unknown check '{args.check}'", file=sys.stderr) - print(f"Available checks: {', '.join(all_checks)}", file=sys.stderr) - sys.exit(2) - - # --check-summary implies --all-evals --summary-only - if args.check_summary: - args.all_evals = True - args.summary_only = True - - if not args.eval_name and not args.all_evals: - parser.error("Either provide an eval_name or use --all-evals") - - repo_root = resolve_repo_root() - - if args.all_evals: - eval_names = get_all_eval_names(repo_root) - check_msg = f" (check: {args.check})" if args.check else "" - print(f"Linting {len(eval_names)} evaluations{check_msg}...\n") - - all_reports = [] - for eval_name in eval_names: - report = lint_evaluation(repo_root, eval_name, check=args.check) - all_reports.append(report) - - if not args.summary_only: - print_report(report) - - # Print overall summary - if args.check_summary: - print_check_summary(all_reports) - else: - print_overall_summary(all_reports) - - passed = sum(1 for r in all_reports if r.passed()) - failed = len(all_reports) - passed - sys.exit(0 if failed == 0 else 1) - - else: - report = lint_evaluation( - repo_root, - args.eval_name, - check=args.check, - ) - print_report(report) - sys.exit(0 if report.passed() else 1) - - -if __name__ == "__main__": - main() diff --git a/tools/run_autolint/__init__.py b/tools/run_autolint/__init__.py deleted file mode 100644 index 64f1866..0000000 --- a/tools/run_autolint/__init__.py +++ /dev/null @@ -1,279 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -""" -Autolint for evaluations - checks against AUTOMATED_CHECKS.md requirements. - -This module checks automatable items from the checklist: -- File structure (location, __init__.py exports, registry, eval.yaml) -- Code quality (imports, constants usage, dependencies) -- Test structure (pytest marks, HF dataset utilities, E2E tests) - -Usage: - python tools/run_autolint.py - python tools/run_autolint.py gpqa - python tools/run_autolint.py --all-evals # Check all evaluations - python tools/run_autolint.py gpqa --check registry # Run only one check - -Suppression: - Line-level: # noautolint: - File-level: # noautolint-file: (at top of file) - Dir-level: Create .noautolint file in a subdirectory with check names (one per line) - Eval-level: Create .noautolint file in eval directory with check names (one per line) - - Example suppressions: - from inspect_ai._util import something # noautolint: private_api_imports - # noautolint-file: private_api_imports - # Or place a .noautolint file in a subdirectory to suppress checks for all files in it -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from tools.run_autolint.checks.best_practices import ( - check_get_model_location, - check_sample_ids, - check_task_overridable_defaults, -) -from tools.run_autolint.checks.code_quality import ( - check_private_api_imports, - check_score_constants, -) -from tools.run_autolint.checks.dependencies import ( - check_external_dependencies, -) -from tools.run_autolint.checks.file_structure import ( - check_eval_location, - check_eval_yaml, - check_init_exports, - check_main_file, - check_readme, - check_registry, - get_eval_path, -) -from tools.run_autolint.checks.tests import ( - check_custom_scorer_tests, - check_custom_solver_tests, - check_custom_tool_tests, - check_e2e_test, - check_record_to_sample_test, - check_tests_exist, - check_tests_init, - get_test_path, -) -from tools.run_autolint.models import LintReport -from tools.run_autolint.suppressions import apply_suppressions, load_suppressions - -# Registry of all available checks with their names and required arguments -# Each check is categorized by what arguments it needs -CheckFunction = Callable[..., None] - -# Checks that only need eval_path and report -EVAL_PATH_CHECKS: dict[str, CheckFunction] = { - "readme": check_readme, - "private_api_imports": check_private_api_imports, - "score_constants": check_score_constants, - # Best practices checks (eval_path only) - "get_model_location": check_get_model_location, - "sample_ids": check_sample_ids, - "task_overridable_defaults": check_task_overridable_defaults, -} - -# Checks that need repo_root, eval_name, and report -REPO_EVAL_CHECKS: dict[str, CheckFunction] = { - "registry": check_registry, - "eval_yaml": check_eval_yaml, -} - -# Checks that need repo_root, eval_name, eval_path, and report -REPO_EVAL_PATH_CHECKS: dict[str, CheckFunction] = { - "external_dependencies": check_external_dependencies, -} - -# Checks that only need test_path and report -TEST_PATH_CHECKS: dict[str, CheckFunction] = { - "e2e_test": check_e2e_test, - "tests_init": check_tests_init, -} - -# Checks that need test_path, eval_path, and report -TEST_EVAL_CHECKS: dict[str, CheckFunction] = { - "record_to_sample_test": check_record_to_sample_test, - "custom_solver_tests": check_custom_solver_tests, - "custom_scorer_tests": check_custom_scorer_tests, - "custom_tool_tests": check_custom_tool_tests, -} - -# Special checks with unique signatures -SPECIAL_CHECKS = {"eval_location", "main_file", "init_exports", "tests_exist"} - - -def get_all_check_names() -> list[str]: - """Get all available check names.""" - names = list(EVAL_PATH_CHECKS.keys()) - names.extend(REPO_EVAL_CHECKS.keys()) - names.extend(REPO_EVAL_PATH_CHECKS.keys()) - names.extend(TEST_PATH_CHECKS.keys()) - names.extend(TEST_EVAL_CHECKS.keys()) - names.extend(SPECIAL_CHECKS) - return sorted(names) - - -def resolve_repo_root() -> Path: - """Resolve the repository root directory.""" - return Path(__file__).resolve().parent.parent.parent - - -# Directories in src/ that are not evaluations (template layout puts evals -# directly under src/, alongside utils/ and examples/). -NON_EVAL_DIRECTORIES = {"utils", "examples", "gdm_capabilities"} - - -def get_all_eval_names(repo_root: Path) -> list[str]: - """Get all evaluation names from the src/ directory.""" - evals_dir = repo_root / "src" - eval_names = [] - - for item in evals_dir.iterdir(): - if ( - item.is_dir() - and not item.name.startswith("_") - and not item.name.startswith(".") - and item.name not in NON_EVAL_DIRECTORIES - ): - # Check if it has an __init__.py (is a Python package) - if (item / "__init__.py").exists(): - eval_names.append(item.name) - - return sorted(eval_names) - - -def _should_run_check( - check_name: str, - filter_check: str | None, -) -> bool: - """Determine if a check should run based on the filter. - - Args: - check_name: Name of the check to run. - filter_check: If set, only run this specific check. - - Returns: - True if the check should run, False otherwise. - """ - if filter_check is not None and check_name != filter_check: - return False - return True - - -def lint_evaluation( - repo_root: Path, - eval_name: str, - check: str | None = None, -) -> LintReport: - """Run lint checks on an evaluation. - - Args: - repo_root: Path to the repository root. - eval_name: Name of the evaluation to lint. - check: Optional specific check to run. If None, runs all checks. - - Returns: - LintReport with the results. - """ - report = LintReport(eval_name=eval_name) - - # Skip directories that are not evaluations (e.g., utility modules) - if eval_name in NON_EVAL_DIRECTORIES: - from tools.run_autolint.models import LintResult - - report.add( - LintResult( - name="non_eval_directory", - status="skip", - message=f"'{eval_name}' is a utility module, not an evaluation", - ) - ) - return report - - # Validate check name if provided - if check is not None: - all_checks = get_all_check_names() - if check not in all_checks: - from tools.run_autolint.models import LintResult - - report.add( - LintResult( - name="invalid_check", - status="fail", - message=f"Unknown check: '{check}'. Available checks: {all_checks}", - ) - ) - return report - - # Always run eval_location first as other checks depend on it - if _should_run_check("eval_location", check): - eval_path = check_eval_location(repo_root, eval_name, report) - else: - # Still need to get eval_path for other checks - eval_path = get_eval_path(repo_root, eval_name) - - if eval_path is None: - return report - - # Load suppressions (applied after all checks run) - suppressions = load_suppressions(eval_path) - - # Run main_file check - if _should_run_check("main_file", check): - check_main_file(eval_path, eval_name, report) - - # Run init_exports check - if _should_run_check("init_exports", check): - check_init_exports(eval_path, eval_name, report) - - # Run eval_path checks - for name, check_func in EVAL_PATH_CHECKS.items(): - if _should_run_check(name, check): - check_func(eval_path, report) - - # Run repo/eval checks - for name, check_func in REPO_EVAL_CHECKS.items(): - if _should_run_check(name, check): - check_func(repo_root, eval_name, report) - - # Run repo/eval/path checks - for name, check_func in REPO_EVAL_PATH_CHECKS.items(): - if _should_run_check(name, check): - check_func(repo_root, eval_name, eval_path, report) - - # Get test path for test checks - if _should_run_check("tests_exist", check): - test_path = check_tests_exist(repo_root, eval_name, report) - else: - # Still need to get test_path for other checks - test_path = get_test_path(repo_root, eval_name) - - # Run test_path checks - for name, check_func in TEST_PATH_CHECKS.items(): - if _should_run_check(name, check): - check_func(test_path, report) - - # Run test/eval checks - for name, check_func in TEST_EVAL_CHECKS.items(): - if _should_run_check(name, check): - check_func(test_path, eval_path, report) - - # Apply suppressions (changes fail/warn to suppressed based on .noautolint and inline comments) - apply_suppressions(report.results, suppressions) - - return report - - -__all__ = [ - "get_all_eval_names", - "get_all_check_names", - "lint_evaluation", - "resolve_repo_root", - "LintReport", -] diff --git a/tools/run_autolint/checks/__init__.py b/tools/run_autolint/checks/__init__.py deleted file mode 100644 index 42054b7..0000000 --- a/tools/run_autolint/checks/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Check modules for autolint.""" - -from tools.run_autolint.checks.best_practices import ( - check_get_model_location, - check_sample_ids, - check_task_overridable_defaults, -) -from tools.run_autolint.checks.code_quality import ( - check_private_api_imports, - check_score_constants, -) -from tools.run_autolint.checks.dependencies import ( - check_external_dependencies, -) -from tools.run_autolint.checks.file_structure import ( - check_eval_location, - check_eval_yaml, - check_init_exports, - check_main_file, - check_readme, - check_registry, -) -from tools.run_autolint.checks.tests import ( - check_custom_scorer_tests, - check_custom_solver_tests, - check_custom_tool_tests, - check_e2e_test, - check_record_to_sample_test, - check_tests_exist, - check_tests_init, -) - -__all__ = [ - # File structure checks - "check_eval_location", - "check_main_file", - "check_init_exports", - "check_registry", - "check_eval_yaml", - "check_readme", - # Code quality checks - "check_private_api_imports", - "check_score_constants", - # Dependency checks - "check_external_dependencies", - # Test checks - "check_tests_exist", - "check_tests_init", - "check_e2e_test", - "check_record_to_sample_test", - "check_custom_solver_tests", - "check_custom_scorer_tests", - "check_custom_tool_tests", - # Best practices checks - "check_get_model_location", - "check_sample_ids", - "check_task_overridable_defaults", -] diff --git a/tools/run_autolint/checks/best_practices.py b/tools/run_autolint/checks/best_practices.py deleted file mode 100644 index bb9d65f..0000000 --- a/tools/run_autolint/checks/best_practices.py +++ /dev/null @@ -1,323 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Best practices checks for autolint. - -These checks verify adherence to BEST_PRACTICES.md guidelines. -""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from tools.run_autolint.checks.utils import ( - Issue, - add_parse_errors_to_report, - parse_python_files, -) -from tools.run_autolint.models import LintReport, LintResult - - -def _get_decorator_name(decorator: ast.expr) -> str | None: - """Extract the name from a decorator AST node. - - Handles: - - @decorator -> "decorator" - - @decorator() -> "decorator" - - @module.decorator -> "decorator" - - @module.decorator() -> "decorator" - """ - if isinstance(decorator, ast.Name): - return decorator.id - elif isinstance(decorator, ast.Call): - if isinstance(decorator.func, ast.Name): - return decorator.func.id - elif isinstance(decorator.func, ast.Attribute): - return decorator.func.attr - elif isinstance(decorator, ast.Attribute): - return decorator.attr - return None - - -def get_call_name(node: ast.Call) -> str | None: - """Extract the function name from a Call AST node. - - Handles: - - foo() -> "foo" - - obj.foo() -> "foo" - - Returns None for complex expressions like subscripts or chained calls. - """ - if isinstance(node.func, ast.Name): - return node.func.id - elif isinstance(node.func, ast.Attribute): - return node.func.attr - return None - - -class GetModelVisitor(ast.NodeVisitor): - """AST visitor to find get_model() calls and track their context.""" - - def __init__(self) -> None: - self.calls: list[tuple[int, str, bool]] = [] # (line, context, is_valid) - self._in_solver_or_scorer = False - self._current_context = "module" - - def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: - """Visit function definition, tracking if it's decorated with @solver or @scorer.""" - old_context = self._current_context - old_in_solver_or_scorer = self._in_solver_or_scorer - - self._current_context = node.name - - # Check if function is decorated with @solver or @scorer - for decorator in node.decorator_list: - if _get_decorator_name(decorator) in ("solver", "scorer"): - self._in_solver_or_scorer = True - break - - self.generic_visit(node) - - self._current_context = old_context - self._in_solver_or_scorer = old_in_solver_or_scorer - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_Call(self, node: ast.Call) -> None: - """Visit function call, checking for get_model().""" - call_name = get_call_name(node) - if call_name == "get_model": - self.calls.append( - (node.lineno, self._current_context, self._in_solver_or_scorer) - ) - self.generic_visit(node) - - -def check_get_model_location(eval_path: Path, report: LintReport) -> None: - """Check that get_model() is only called inside solvers or scorers. - - Per BEST_PRACTICES.md: "Resolve concrete models late (inside @solver/@scorer) - so tasks remain declarative and configurable." - """ - issues: list[Issue] = [] - - parse_results = parse_python_files(eval_path) - if add_parse_errors_to_report( - "get_model_location", parse_results.failed_paths, report - ): - return - - for result in parse_results.successful: - visitor = GetModelVisitor() - visitor.visit(result.tree) # type: ignore[arg-type] # tree is non-None for successful results - - for line, context, is_valid in visitor.calls: - if not is_valid: - issues.append(Issue(str(result.path), line, context)) - - if issues: - # One result per call site so line-level `# noautolint: get_model_location` works. - for issue in issues: - report.add( - LintResult( - name="get_model_location", - status="warn", - message=( - "get_model() called outside @solver/@scorer. " - "Resolve models inside @solver/@scorer so tasks stay declarative." - + (f" ({issue.detail})" if issue.detail else "") - ), - file=issue.file, - line=issue.line, - ) - ) - else: - report.add( - LintResult( - name="get_model_location", - status="pass", - message="get_model() calls are properly inside @solver/@scorer decorated functions", - ) - ) - - -class TaskParameterVisitor(ast.NodeVisitor): - """AST visitor to find @task decorated functions and their parameters.""" - - def __init__(self) -> None: - self.tasks: list[tuple[str, int, set[str]]] = [] # (name, line, params) - - def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - """Visit function definition, checking for @task decorator.""" - is_task = any(_get_decorator_name(d) == "task" for d in node.decorator_list) - - if is_task: - params = set() - for arg in node.args.args: - params.add(arg.arg) - for arg in node.args.kwonlyargs: - params.add(arg.arg) - self.tasks.append((node.name, node.lineno, params)) - - self.generic_visit(node) - - -class SampleIdVisitor(ast.NodeVisitor): - """AST visitor to find Sample() calls and check for id= parameter.""" - - def __init__(self) -> None: - self.samples: list[tuple[int, bool]] = [] # (line, has_id) - - def visit_Call(self, node: ast.Call) -> None: - """Visit function call, checking for Sample() constructor.""" - call_name = get_call_name(node) - if call_name == "Sample": - has_id = False - for keyword in node.keywords: - if keyword.arg == "id": - has_id = True - break - self.samples.append((node.lineno, has_id)) - - self.generic_visit(node) - - -def check_sample_ids(eval_path: Path, report: LintReport) -> None: - """Check if Sample() calls include an id parameter. - - Per BEST_PRACTICES.md: "Use stable, canonical IDs for samples." - """ - samples_without_id = [] - total_samples = 0 - - parse_results = parse_python_files(eval_path) - if add_parse_errors_to_report("sample_ids", parse_results.failed_paths, report): - return - - for result in parse_results.successful: - visitor = SampleIdVisitor() - visitor.visit(result.tree) # type: ignore[arg-type] # tree is non-None for successful results - - for line, has_id in visitor.samples: - total_samples += 1 - if not has_id: - samples_without_id.append((str(result.path.name), line)) - - if total_samples == 0: - report.add( - LintResult( - name="sample_ids", - status="skip", - message="No Sample() calls found", - ) - ) - return - - if samples_without_id: - report.add( - LintResult( - name="sample_ids", - status="fail", - message=f"Sample() calls without id= parameter: {samples_without_id[:5]}", - ) - ) - else: - report.add( - LintResult( - name="sample_ids", - status="pass", - message=f"All {total_samples} Sample() calls include id parameter", - ) - ) - - -class TaskDefaultsVisitor(ast.NodeVisitor): - """AST visitor to check @task function signatures for optional parameters.""" - - def __init__(self) -> None: - self.tasks: list[ - tuple[str, int, dict[str, bool]] - ] = [] # (name, line, param_defaults) - - def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - """Visit function definition, checking for @task decorator.""" - is_task = any(_get_decorator_name(d) == "task" for d in node.decorator_list) - - if is_task: - param_defaults: dict[str, bool] = {} - - # Check positional args with defaults - num_defaults = len(node.args.defaults) - num_args = len(node.args.args) - args_with_defaults = ( - node.args.args[num_args - num_defaults :] if num_defaults > 0 else [] - ) - - for arg in node.args.args: - param_defaults[arg.arg] = arg in args_with_defaults - - # Check keyword-only args - for i, arg in enumerate(node.args.kwonlyargs): - has_default = node.args.kw_defaults[i] is not None - param_defaults[arg.arg] = has_default - - self.tasks.append((node.name, node.lineno, param_defaults)) - - self.generic_visit(node) - - -def check_task_overridable_defaults(eval_path: Path, report: LintReport) -> None: - """Check if @task functions have defaults for key parameters. - - Per BEST_PRACTICES.md: "Provide defaults and allow overrides for solvers, - scorers, metrics, and grader models." - """ - overridable_params = {"solver", "scorer", "metric", "metrics", "grader", "model"} - issues = [] - total_tasks = 0 - - parse_results = parse_python_files(eval_path) - if add_parse_errors_to_report( - "task_overridable_defaults", parse_results.failed_paths, report - ): - return - - for result in parse_results.successful: - visitor = TaskDefaultsVisitor() - visitor.visit(result.tree) # type: ignore[arg-type] # tree is non-None for successful results - - for task_name, _, param_defaults in visitor.tasks: - total_tasks += 1 - missing_defaults = [] - for param_name in overridable_params: - for actual_param, has_default in param_defaults.items(): - if param_name in actual_param.lower() and not has_default: - missing_defaults.append(actual_param) - if missing_defaults: - issues.append((task_name, missing_defaults)) - - if total_tasks == 0: - report.add( - LintResult( - name="task_overridable_defaults", - status="skip", - message="No @task decorated functions found", - ) - ) - return - - if issues: - report.add( - LintResult( - name="task_overridable_defaults", - status="fail", - message=f"Tasks with overridable params lacking defaults: {issues[:3]}", - ) - ) - else: - report.add( - LintResult( - name="task_overridable_defaults", - status="pass", - message="Tasks provide defaults for overridable parameters", - ) - ) diff --git a/tools/run_autolint/checks/code_quality.py b/tools/run_autolint/checks/code_quality.py deleted file mode 100644 index 8c7b2ba..0000000 --- a/tools/run_autolint/checks/code_quality.py +++ /dev/null @@ -1,98 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Code quality checks for autolint.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from tools.run_autolint.checks.best_practices import get_call_name -from tools.run_autolint.checks.utils import ( - Issue, - add_parse_errors_to_report, - parse_python_files, -) -from tools.run_autolint.models import LintReport, LintResult - - -def check_private_api_imports(eval_path: Path, report: LintReport) -> None: - """Check for imports from private (_prefixed) inspect_ai modules.""" - issues: list[Issue] = [] - - parse_results = parse_python_files(eval_path) - if add_parse_errors_to_report( - "private_api_imports", parse_results.failed_paths, report - ): - return - - for result in parse_results.successful: - for node in ast.walk(result.tree): # type: ignore[arg-type] # tree is non-None for successful results - if isinstance(node, ast.ImportFrom) and node.module: - # Check for inspect_ai.*._* pattern - if node.module.startswith("inspect_ai.") and "._" in node.module: - issues.append(Issue(str(result.path), node.lineno, node.module)) - - if issues: - for issue in issues: - report.add( - LintResult( - name="private_api_imports", - status="fail", - message=f"Import from private inspect_ai module: {issue.detail}", - file=issue.file, - line=issue.line, - ) - ) - else: - report.add( - LintResult( - name="private_api_imports", - status="pass", - message="No private API imports found", - ) - ) - - -def check_score_constants(eval_path: Path, report: LintReport) -> None: - """Check that Score() uses CORRECT/INCORRECT constants instead of literals.""" - issues: list[Issue] = [] - - parse_results = parse_python_files(eval_path) - if add_parse_errors_to_report( - "score_constants", parse_results.failed_paths, report - ): - return - - for result in parse_results.successful: - for node in ast.walk(result.tree): # type: ignore[arg-type] # tree is non-None for successful results - if isinstance(node, ast.Call): - # Check if it's a Score() call - if get_call_name(node) == "Score": - for keyword in node.keywords: - if keyword.arg == "value": - # Check if value is a string literal "C" or "I" - if isinstance(keyword.value, ast.Constant): - if keyword.value.value in ( - "C", - "I", - "CORRECT", - "INCORRECT", - ): - issues.append(Issue(str(result.path), node.lineno)) - - if issues: - report.add( - LintResult( - name="score_constants", - status="fail", - message=f"Found {len(issues)} Score() calls with literal strings - consider using CORRECT/INCORRECT constants", - ) - ) - else: - report.add( - LintResult( - name="score_constants", - status="pass", - message="Score() calls appear to use constants or computed values", - ) - ) diff --git a/tools/run_autolint/checks/dependencies.py b/tools/run_autolint/checks/dependencies.py deleted file mode 100644 index d53d567..0000000 --- a/tools/run_autolint/checks/dependencies.py +++ /dev/null @@ -1,308 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Dependency checks for autolint.""" - -from __future__ import annotations - -import ast -import functools -import re -import sys -from importlib.metadata import packages_distributions -from pathlib import Path - -import toml -from tools.run_autolint.checks.utils import add_parse_errors_to_report -from tools.run_autolint.models import LintReport, LintResult - - -def _extract_package_name(dep: str) -> str | None: - """Extract package name from a dependency string. - - Removes version specifiers, extras, and environment markers. - Returns lowercase package name, or None if empty. - """ - dep_name = re.split(r"[<>=;[\s]", dep)[0].strip().lower() - return dep_name if dep_name else None - - -@functools.cache -def _get_stdlib_modules() -> frozenset[str]: - """Get standard library module names. - - Uses sys.stdlib_module_names (Python 3.10+) plus some common additions. - """ - # sys.stdlib_module_names is available in Python 3.10+ - stdlib = set(sys.stdlib_module_names) - # Add typing_extensions as it's a common backport that's often available - stdlib.add("typing_extensions") - return frozenset(stdlib) - - -@functools.cache -def _get_core_dependencies(repo_root: Path) -> frozenset[str]: - """Load core dependencies from pyproject.toml [project].dependencies.""" - pyproject_path = repo_root / "pyproject.toml" - if not pyproject_path.exists(): - return frozenset() - - data = toml.load(pyproject_path) - deps = data.get("project", {}).get("dependencies", []) - - core_deps = set() - for dep in deps: - dep_name = _extract_package_name(dep) - if dep_name: - core_deps.add(dep_name) - - return frozenset(core_deps) - - -# Well-known import name → package name aliases for packages that may not be -# installed (optional dependencies). importlib.metadata can only resolve -# mappings for *installed* packages, so we need a static fallback for common -# cases where the import name differs from the PyPI package name. -_KNOWN_IMPORT_ALIASES: dict[str, str] = { - "sklearn": "scikit-learn", - "PIL": "pillow", - "yaml": "pyyaml", - "bs4": "beautifulsoup4", - "odf": "odfpy", -} - - -@functools.cache -def _get_import_to_package_map() -> dict[str, str]: - """Build mapping from import names to package names using importlib.metadata. - - Uses packages_distributions() which returns a mapping of top-level import - names to the distribution packages that provide them. Falls back to - ``_KNOWN_IMPORT_ALIASES`` for packages that are not installed. - """ - mapping: dict[str, str] = dict(_KNOWN_IMPORT_ALIASES) - for import_name, distributions in packages_distributions().items(): - if distributions: - # Use the first distribution (most packages have only one) - mapping[import_name] = distributions[0] - return mapping - - -def _get_imports_from_file(file_path: Path) -> tuple[set[str], str | None]: - """Extract top-level import module names from a Python file. - - Returns: - Tuple of (imports set, error message or None). - """ - imports: set[str] = set() - try: - tree = ast.parse(file_path.read_text()) - except SyntaxError as e: - return set(), str(e) - - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - # Get the top-level module name - top_module = alias.name.split(".")[0] - imports.add(top_module) - elif isinstance(node, ast.ImportFrom) and node.module: - # Get the top-level module name - top_module = node.module.split(".")[0] - imports.add(top_module) - - return imports, None - - -def _get_noautolint_subdirectories(eval_path: Path) -> set[Path]: - """Find subdirectories with .noautolint files that should skip all checks. - - Args: - eval_path: Root evaluation directory to search. - - Returns: - Set of subdirectories (excluding root) that should be skipped. - """ - skip_dirs: set[Path] = set() - for noautolint_file in eval_path.rglob(".noautolint"): - # Skip the root eval directory - its .noautolint uses per-check suppressions - if noautolint_file.parent != eval_path: - skip_dirs.add(noautolint_file.parent) - return skip_dirs - - -def _is_path_under_any(path: Path, skip_dirs: set[Path]) -> bool: - """Check if a path is under any of the skip directories.""" - return any(path.is_relative_to(skip_dir) for skip_dir in skip_dirs) - - -def _get_all_imports_from_eval(eval_path: Path) -> tuple[set[str], list[str]]: - """Get all imports from an evaluation's Python files. - - Skips subdirectories that contain a .noautolint file. - - Returns: - Tuple of (all imports set, list of files with syntax errors). - """ - all_imports: set[str] = set() - syntax_error_files: list[str] = [] - - # Find subdirectories with .noautolint files to skip - skip_dirs = _get_noautolint_subdirectories(eval_path) - - for py_file in eval_path.rglob("*.py"): - # Skip files in directories with their own .noautolint - if skip_dirs and _is_path_under_any(py_file, skip_dirs): - continue - imports, error = _get_imports_from_file(py_file) - if error: - syntax_error_files.append(str(py_file)) - else: - all_imports.update(imports) - return all_imports, syntax_error_files - - -def _get_local_modules(eval_path: Path) -> set[str]: - """Get module names that are local to the evaluation directory.""" - local_modules = set() - - # Add all Python files and directories in the eval and its subdirectories - for py_file in eval_path.rglob("*.py"): - # Add module name (filename without .py) - local_modules.add(py_file.stem) - # Add all parent directory names up to eval_path - parent = py_file.parent - while parent not in (eval_path, eval_path.parent): - local_modules.add(parent.name) - parent = parent.parent - - # Add the eval's own package name - local_modules.add(eval_path.name) - return local_modules - - -@functools.cache -def _load_pyproject_optional_deps(repo_root: Path) -> dict[str, frozenset[str]]: - """Load optional dependencies from pyproject.toml. - - Checks both [project.optional-dependencies] and [dependency-groups] sections. - """ - pyproject_path = repo_root / "pyproject.toml" - if not pyproject_path.exists(): - return {} - - data = toml.load(pyproject_path) - - optional_deps: dict[str, frozenset[str]] = {} - - # Load from [project.optional-dependencies] - raw_optional_deps = data.get("project", {}).get("optional-dependencies", {}) - for group_name, deps_list in raw_optional_deps.items(): - deps = { - dep_name for dep in deps_list if (dep_name := _extract_package_name(dep)) - } - optional_deps[group_name] = frozenset(deps) - - # Also load from [dependency-groups] (excluding 'dev' which is for development) - raw_dep_groups = data.get("dependency-groups", {}) - for group_name, deps_list in raw_dep_groups.items(): - if group_name == "dev": - continue # Skip dev dependencies - if isinstance(deps_list, list): - deps = { - dep_name - for dep in deps_list - if isinstance(dep, str) and (dep_name := _extract_package_name(dep)) - } - optional_deps[group_name] = optional_deps.get( - group_name, frozenset() - ) | frozenset(deps) - - return optional_deps - - -def check_external_dependencies( - repo_root: Path, eval_name: str, eval_path: Path, report: LintReport -) -> None: - """Check if external imports have corresponding optional dependencies in pyproject.toml.""" - # Get all imports from the evaluation - imports, syntax_error_files = _get_all_imports_from_eval(eval_path) - if add_parse_errors_to_report("external_dependencies", syntax_error_files, report): - return - - # Get local modules that shouldn't be counted as external - local_modules = _get_local_modules(eval_path) - - # Get stdlib and core dependencies - stdlib_modules = _get_stdlib_modules() - core_deps = _get_core_dependencies(repo_root) - - # Get import→package mapping to resolve import names to package names - # (e.g., PIL → pillow). This avoids needing manual alias maintenance. - import_to_package = _get_import_to_package_map() - - # Filter to external imports (not stdlib or core) - external_imports = set() - for imp in imports: - imp_lower = imp.lower() - # Resolve import name to package name (e.g., PIL → pillow) - package_name = import_to_package.get(imp, imp).lower() - if imp_lower not in stdlib_modules and package_name not in core_deps: - # Skip imports from inspect_evals itself - if imp != "inspect_evals": - # Skip local modules - if imp not in local_modules and imp_lower not in local_modules: - external_imports.add(imp) - - if not external_imports: - report.add( - LintResult( - name="external_dependencies", - status="pass", - message="No external dependencies detected beyond core requirements", - ) - ) - return - - # Load optional dependencies from pyproject.toml - optional_deps = _load_pyproject_optional_deps(repo_root) - - # Check deps that might be in any optional dependency groups - all_optional_deps: set[str] = set() - for deps in optional_deps.values(): - all_optional_deps.update(deps) - - # Check for missing dependencies - missing_deps = [] - for imp in external_imports: - # Get package name from import name - package_name = import_to_package.get(imp, imp).lower() - imp_lower = imp.lower() - - # Check if the package is declared - if package_name not in all_optional_deps and imp_lower not in all_optional_deps: - missing_deps.append(f"{imp} (package: {package_name})") - - if missing_deps: - report.add( - LintResult( - name="external_dependencies", - status="fail", - message=f"External imports may need pyproject.toml optional-dependencies: {missing_deps[:5]}", - ) - ) - elif external_imports and eval_name not in optional_deps: - # Check if eval should have its own group - report.add( - LintResult( - name="external_dependencies", - status="fail", - message=f"Evaluation uses external packages ({list(external_imports)[:3]}...) but has no dedicated optional-dependency group", - ) - ) - else: - report.add( - LintResult( - name="external_dependencies", - status="pass", - message=f"External dependencies appear to be declared (imports: {len(external_imports)})", - ) - ) diff --git a/tools/run_autolint/checks/file_structure.py b/tools/run_autolint/checks/file_structure.py deleted file mode 100644 index d218adc..0000000 --- a/tools/run_autolint/checks/file_structure.py +++ /dev/null @@ -1,360 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""File structure checks for autolint.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -import yaml -from tools.run_autolint.checks.best_practices import _get_decorator_name -from tools.run_autolint.models import LintReport, LintResult - - -def _find_task_functions(file_path: Path) -> list[str]: - """Find all @task decorated function names in a Python file.""" - if not file_path.exists(): - return [] - - try: - tree = ast.parse(file_path.read_text()) - except SyntaxError as e: - raise Exception(f"Failed to parse @task decorations in {file_path}: {e}") - - task_functions = [] - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - if any(_get_decorator_name(d) == "task" for d in node.decorator_list): - task_functions.append(node.name) - - return task_functions - - -def _get_exported_names(init_file: Path) -> set[str]: - """Get all names exported from an __init__.py file.""" - if not init_file.exists(): - return set() - - try: - tree = ast.parse(init_file.read_text()) - except SyntaxError as e: - raise Exception(f"Failed to parse exported names in {init_file}: {e}") - - exported = set() - - for node in ast.walk(tree): - # Check __all__ = [...] - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id == "__all__": - if isinstance(node.value, (ast.List, ast.Tuple)): - for elt in node.value.elts: - if isinstance(elt, ast.Constant) and isinstance( - elt.value, str - ): - exported.add(elt.value) - - # Check "from .module import name" or "from .module import name as alias" - if isinstance(node, ast.ImportFrom): - for alias in node.names: - # The exported name is the alias if present, otherwise the original name - exported_name = alias.asname if alias.asname else alias.name - exported.add(exported_name) - - return exported - - -def get_eval_path(repo_root: Path, eval_name: str) -> Path | None: - """Find the evaluation directory.""" - # Template layout: evals live directly under src/, not src/inspect_evals/. - eval_path = repo_root / "src" / eval_name - if eval_path.is_dir(): - return eval_path - return None - - -def check_eval_location( - repo_root: Path, eval_name: str, report: LintReport -) -> Path | None: - """Check that evaluation is in src/.""" - eval_path = get_eval_path(repo_root, eval_name) - if eval_path: - report.add( - LintResult( - name="eval_location", - status="pass", - message=f"Evaluation located at {eval_path.relative_to(repo_root)}", - ) - ) - return eval_path - else: - report.add( - LintResult( - name="eval_location", - status="fail", - message=f"Evaluation directory not found: src/{eval_name}", - ) - ) - return None - - -def check_main_file(eval_path: Path, eval_name: str, report: LintReport) -> list[str]: - """Check that .py exists and has at least one @task function. - - Returns the list of task function names found. - """ - main_file = eval_path / f"{eval_name}.py" - - if not main_file.exists(): - report.add( - LintResult( - name="main_file", - status="fail", - message=f"Missing main file: {eval_name}.py", - file=str(main_file), - ) - ) - return [] - - task_functions = _find_task_functions(main_file) - - if not task_functions: - report.add( - LintResult( - name="main_file", - status="fail", - message=f"{eval_name}.py has no @task decorated functions", - file=str(main_file), - ) - ) - return [] - - report.add( - LintResult( - name="main_file", - status="pass", - message=f"{eval_name}.py has {len(task_functions)} @task function(s): {task_functions}", - file=str(main_file), - ) - ) - return task_functions - - -def check_init_exports(eval_path: Path, eval_name: str, report: LintReport) -> None: - """Check that __init__.py exports task functions from the main file.""" - init_file = eval_path / "__init__.py" - main_file = eval_path / f"{eval_name}.py" - - if not init_file.exists(): - report.add( - LintResult( - name="init_exports", - status="fail", - message="Missing __init__.py file", - file=str(init_file), - ) - ) - return - - try: - ast.parse(init_file.read_text()) - except SyntaxError as e: - report.add( - LintResult( - name="init_exports", - status="fail", - message=f"Syntax error in __init__.py: {e}", - file=str(init_file), - ) - ) - return - - # Find task functions from main file - if not main_file.exists(): - report.add( - LintResult( - name="init_exports", - status="skip", - message=f"Main file {eval_name}.py not found, cannot check exports", - file=str(init_file), - ) - ) - return - - task_functions = _find_task_functions(main_file) - if not task_functions: - report.add( - LintResult( - name="init_exports", - status="skip", - message="No task functions to check for exports", - file=str(init_file), - ) - ) - return - - exported_names = _get_exported_names(init_file) - missing_exports = [fn for fn in task_functions if fn not in exported_names] - - if missing_exports: - report.add( - LintResult( - name="init_exports", - status="fail", - message=f"__init__.py does not export task functions: {missing_exports}", - file=str(init_file), - ) - ) - else: - report.add( - LintResult( - name="init_exports", - status="pass", - message=f"__init__.py exports all {len(task_functions)} task function(s)", - file=str(init_file), - ) - ) - - -def check_registry(repo_root: Path, eval_name: str, report: LintReport) -> None: - """Check that the evaluation is registered as an inspect_ai entry point. - - Template layout uses ``[project.entry-points.inspect_ai]`` in - ``pyproject.toml`` as the registry rather than a Python ``_registry.py``. - """ - pyproject = repo_root / "pyproject.toml" - if not pyproject.exists(): - report.add( - LintResult( - name="registry", - status="skip", - message="pyproject.toml not found", - ) - ) - return - - try: - import tomllib - except ModuleNotFoundError: # Python < 3.11 - import tomli as tomllib # type: ignore[import-not-found] - - with open(pyproject, "rb") as f: - data = tomllib.load(f) - entry_points = data.get("project", {}).get("entry-points", {}).get("inspect_ai", {}) - - if eval_name in entry_points: - report.add( - LintResult( - name="registry", - status="pass", - message=f"Evaluation registered in pyproject.toml as inspect_ai/{eval_name}", - file=str(pyproject), - ) - ) - else: - report.add( - LintResult( - name="registry", - status="fail", - message=( - f"Evaluation not registered. Add to pyproject.toml: " - f'[project.entry-points.inspect_ai]\n{eval_name} = "{eval_name}"' - ), - file=str(pyproject), - ) - ) - - -def check_eval_yaml(repo_root: Path, eval_name: str, report: LintReport) -> None: - """Check that evaluation has an eval.yaml with required fields.""" - eval_yaml_file = repo_root / "src" / eval_name / "eval.yaml" - if not eval_yaml_file.exists(): - report.add( - LintResult( - name="eval_yaml", - status="fail", - message=f"Missing eval.yaml in src/{eval_name}/", - file=str(eval_yaml_file), - ) - ) - return - - with open(eval_yaml_file) as f: - data = yaml.safe_load(f) - - if not isinstance(data, dict): - report.add( - LintResult( - name="eval_yaml", - status="fail", - message="eval.yaml is not a mapping", - file=str(eval_yaml_file), - ) - ) - return - - # Check required fields - missing_fields = [] - for field_name in [ - "title", - "description", - "group", - "contributors", - "tasks", - ]: - if field_name not in data: - missing_fields.append(field_name) - - if missing_fields: - report.add( - LintResult( - name="eval_yaml", - status="fail", - message=f"eval.yaml missing fields: {missing_fields}", - file=str(eval_yaml_file), - ) - ) - else: - report.add( - LintResult( - name="eval_yaml", - status="pass", - message="eval.yaml has all required fields", - file=str(eval_yaml_file), - ) - ) - - -def check_readme(eval_path: Path, report: LintReport) -> None: - """Check that README.md exists.""" - readme_file = eval_path / "README.md" - if readme_file.exists(): - content = readme_file.read_text() - # Check for TODO markers - if "TODO:" in content: - report.add( - LintResult( - name="readme", - status="warn", - message="README.md contains TODO markers", - file=str(readme_file), - ) - ) - else: - report.add( - LintResult( - name="readme", - status="pass", - message="README.md exists", - file=str(readme_file), - ) - ) - else: - report.add( - LintResult( - name="readme", - status="fail", - message="Missing README.md", - file=str(readme_file), - ) - ) diff --git a/tools/run_autolint/checks/tests.py b/tools/run_autolint/checks/tests.py deleted file mode 100644 index 229bb4e..0000000 --- a/tools/run_autolint/checks/tests.py +++ /dev/null @@ -1,369 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Test-related checks for autolint.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from tools.run_autolint.checks.best_practices import _get_decorator_name, get_call_name -from tools.run_autolint.checks.utils import ( - _get_noautolint_directories, - _is_path_under_any, - add_parse_errors_to_report, -) -from tools.run_autolint.models import LintReport, LintResult - - -def get_test_path(repo_root: Path, eval_name: str) -> Path | None: - """Find the test directory for an evaluation.""" - test_path = repo_root / "tests" / eval_name - if test_path.is_dir(): - return test_path - return None - - -def check_tests_exist( - repo_root: Path, eval_name: str, report: LintReport -) -> Path | None: - """Check that test directory exists.""" - test_path = get_test_path(repo_root, eval_name) - if test_path: - report.add( - LintResult( - name="tests_exist", - status="pass", - message=f"Test directory exists at tests/{eval_name}", - ) - ) - return test_path - else: - report.add( - LintResult( - name="tests_exist", - status="fail", - message=f"Missing test directory: tests/{eval_name}", - ) - ) - return None - - -def _get_eval_aliases(tree: ast.AST) -> set[str]: - """Find all local names that alias inspect_ai.eval or eval_async.""" - aliases: set[str] = {"eval", "eval_async"} - for node in ast.walk(tree): - if ( - isinstance(node, ast.ImportFrom) - and node.module - and "inspect_ai" in node.module - ): - for alias in node.names: - if alias.name in ("eval", "eval_async") and alias.asname: - aliases.add(alias.asname) - return aliases - - -def _has_eval_call(tree: ast.AST) -> bool: - """Check if AST contains a call to eval() / eval_async() (or an alias).""" - eval_names = _get_eval_aliases(tree) - for node in ast.walk(tree): - if isinstance(node, ast.Call): - call_name = get_call_name(node) - if call_name in eval_names: - return True - return False - - -def check_e2e_test(test_path: Path | None, report: LintReport) -> None: - """Check for E2E tests that call eval() or eval_async() with mockllm/model.""" - if test_path is None: - report.add( - LintResult( - name="e2e_test", - status="fail", - message="No test directory exists", - ) - ) - return - - found_e2e_test = False - for py_file in test_path.rglob("*.py"): - try: - content = py_file.read_text() - tree = ast.parse(content) - except SyntaxError as e: - raise SyntaxError(f"Syntax error in {py_file}") from e - - # Check if file has both eval() call and mockllm/model - if _has_eval_call(tree) and "mockllm/model" in content: - found_e2e_test = True - break - - if found_e2e_test: - report.add( - LintResult( - name="e2e_test", - status="pass", - message="E2E test with eval() and mockllm/model found", - ) - ) - else: - report.add( - LintResult( - name="e2e_test", - status="fail", - message="No E2E test found (need test file with eval() call and mockllm/model)", - ) - ) - - -def check_record_to_sample_test( - test_path: Path | None, eval_path: Path, report: LintReport -) -> None: - """Check if record_to_sample is tested.""" - if test_path is None: - report.add( - LintResult( - name="record_to_sample_test", - status="fail", - message="No test directory exists", - ) - ) - return - - # First check if eval uses record_to_sample - has_record_to_sample = False - for py_file in eval_path.rglob("*.py"): - content = py_file.read_text() - if "record_to_sample" in content or "def record_to_sample" in content: - has_record_to_sample = True - break - - if not has_record_to_sample: - report.add( - LintResult( - name="record_to_sample_test", - status="skip", - message="Evaluation does not use record_to_sample", - ) - ) - return - - # Check if tests reference it - tested = False - for py_file in test_path.rglob("*.py"): - content = py_file.read_text() - if "record_to_sample" in content: - tested = True - break - - if tested: - report.add( - LintResult( - name="record_to_sample_test", - status="pass", - message="record_to_sample is tested", - ) - ) - else: - report.add( - LintResult( - name="record_to_sample_test", - status="fail", - message="record_to_sample function exists but is not tested", - ) - ) - - -class DecoratedFunctionResults: - """Results from searching for decorated functions.""" - - def __init__(self) -> None: - self.functions: list[tuple[str, str, int]] = [] # (file, name, line) - self.syntax_errors: list[tuple[str, str]] = [] # (file, error_message) - - -def _find_decorated_functions( - eval_path: Path, decorator_name: str -) -> DecoratedFunctionResults: - """Find functions decorated with a specific decorator. - - Returns DecoratedFunctionResults containing: - - functions: list of (file_path, function_name, line_number) - - syntax_errors: list of (file_path, error_message) for unparseable files - - Skips subdirectories containing .noautolint files. - """ - results = DecoratedFunctionResults() - - # Find subdirectories with .noautolint files to skip - # (excluding the root eval directory itself) - all_noautolint_dirs = _get_noautolint_directories(eval_path) - skip_dirs = {d for d in all_noautolint_dirs if d != eval_path} - - for py_file in eval_path.rglob("*.py"): - # Skip files in directories with their own .noautolint - if skip_dirs and _is_path_under_any(py_file, skip_dirs): - continue - - try: - tree = ast.parse(py_file.read_text()) - except SyntaxError as e: - results.syntax_errors.append((str(py_file), str(e))) - continue - - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - if any( - _get_decorator_name(d) == decorator_name - for d in node.decorator_list - ): - results.functions.append((str(py_file), node.name, node.lineno)) - return results - - -def _function_tested(test_path: Path, function_name: str) -> bool: - """Check if a function is referenced in tests.""" - for py_file in test_path.rglob("*.py"): - try: - content = py_file.read_text() - except (OSError, UnicodeDecodeError) as e: - raise Exception(f"Could not read {py_file}, got error {e}") - if function_name in content: - return True - return False - - -def _check_custom_decorated_tests( - test_path: Path | None, - eval_path: Path, - report: LintReport, - decorator_type: str, -) -> None: - """Check if custom decorated functions have corresponding tests. - - Args: - test_path: Path to test directory (or None if it doesn't exist). - eval_path: Path to evaluation directory. - report: LintReport to add results to. - decorator_type: The decorator name (e.g., "solver", "scorer", "tool"). - """ - check_name = f"custom_{decorator_type}_tests" - plural = f"{decorator_type}s" - - if test_path is None: - report.add( - LintResult( - name=check_name, - status="fail", - message="No test directory exists", - ) - ) - return - - search_results = _find_decorated_functions(eval_path, decorator_type) - error_files = [path for path, _ in search_results.syntax_errors] - if add_parse_errors_to_report(check_name, error_files, report): - return - - if not search_results.functions: - report.add( - LintResult( - name=check_name, - status="skip", - message=f"No custom {plural} found", - ) - ) - return - - untested = [ - func_name - for _, func_name, _ in search_results.functions - if not _function_tested(test_path, func_name) - ] - - if untested: - report.add( - LintResult( - name=check_name, - status="fail", - message=f"Custom {plural} without apparent tests: {untested[:5]}", - ) - ) - else: - report.add( - LintResult( - name=check_name, - status="pass", - message=f"All {len(search_results.functions)} custom {plural} appear tested", - ) - ) - - -def check_custom_solver_tests( - test_path: Path | None, eval_path: Path, report: LintReport -) -> None: - """Check if custom solvers have corresponding tests.""" - _check_custom_decorated_tests(test_path, eval_path, report, "solver") - - -def check_custom_scorer_tests( - test_path: Path | None, eval_path: Path, report: LintReport -) -> None: - """Check if custom scorers have corresponding tests.""" - _check_custom_decorated_tests(test_path, eval_path, report, "scorer") - - -def check_custom_tool_tests( - test_path: Path | None, eval_path: Path, report: LintReport -) -> None: - """Check if custom tools have corresponding tests.""" - _check_custom_decorated_tests(test_path, eval_path, report, "tool") - - -# Directories to exclude when checking for __init__.py -EXCLUDED_TEST_DIRS = {"__pycache__", ".mypy_cache", "utils"} - - -def check_tests_init(test_path: Path | None, report: LintReport) -> None: - """Check that test directory and subdirectories have __init__.py files.""" - if test_path is None: - report.add( - LintResult( - name="tests_init", - status="fail", - message="No test directory exists", - ) - ) - return - - missing_init: list[str] = [] - - # Check if test_path itself has __init__.py - if not (test_path / "__init__.py").exists(): - missing_init.append(test_path.name) - - # Check subdirectories recursively (excluding special dirs) - for item in test_path.rglob("*"): - if item.is_dir() and item.name not in EXCLUDED_TEST_DIRS: - if not (item / "__init__.py").exists(): - # Get relative path from test_path for clearer messaging - rel_path = item.relative_to(test_path) - missing_init.append(str(rel_path)) - - if missing_init: - report.add( - LintResult( - name="tests_init", - status="fail", - message=f"Test directories missing __init__.py: {missing_init}", - ) - ) - else: - report.add( - LintResult( - name="tests_init", - status="pass", - message="Test directory has __init__.py", - ) - ) diff --git a/tools/run_autolint/checks/utils.py b/tools/run_autolint/checks/utils.py deleted file mode 100644 index 78b3dc2..0000000 --- a/tools/run_autolint/checks/utils.py +++ /dev/null @@ -1,164 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Shared utilities for autolint checks.""" - -from __future__ import annotations - -import ast -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from tools.run_autolint.models import LintReport - - -@dataclass -class Issue: - """A lint issue found during a check.""" - - file: str - line: int - detail: str = "" - - def __str__(self) -> str: - if self.detail: - return f"{self.file}:{self.line} ({self.detail})" - return f"{self.file}:{self.line}" - - -@dataclass -class ParseResult: - """Result of parsing a Python file.""" - - path: Path - tree: ast.AST | None = None - error: str | None = None - - @property - def success(self) -> bool: - return self.tree is not None - - -@dataclass -class ParseResults: - """Collection of parse results from multiple files.""" - - results: list[ParseResult] = field(default_factory=list) - - @property - def successful(self) -> list[ParseResult]: - """Get all successfully parsed results.""" - return [r for r in self.results if r.success] - - @property - def failed(self) -> list[ParseResult]: - """Get all failed parse results.""" - return [r for r in self.results if not r.success] - - @property - def failed_paths(self) -> list[str]: - """Get paths of all files that failed to parse.""" - return [str(r.path) for r in self.failed] - - -def safe_parse_file(file_path: Path) -> ParseResult: - """Parse a Python file, returning None tree on syntax error instead of raising.""" - try: - tree = ast.parse(file_path.read_text()) - return ParseResult(path=file_path, tree=tree) - except SyntaxError as e: - return ParseResult(path=file_path, error=str(e)) - - -def _get_noautolint_directories(directory: Path) -> set[Path]: - """Find directories with .noautolint files that skip all checks. - - A .noautolint file in a subdirectory skips all checks for files in that directory - and its subdirectories. - - Args: - directory: Root directory to search. - - Returns: - Set of directories that should be skipped. - """ - skip_dirs: set[Path] = set() - for noautolint_file in directory.rglob(".noautolint"): - skip_dirs.add(noautolint_file.parent) - return skip_dirs - - -def _is_path_under_any(path: Path, skip_dirs: set[Path]) -> bool: - """Check if a path is under any of the skip directories. - - Args: - path: Path to check. - skip_dirs: Set of directories to skip. - - Returns: - True if path is in or under any skip directory. - """ - return any(path.is_relative_to(skip_dir) for skip_dir in skip_dirs) - - -def parse_python_files( - directory: Path, skip_subdirs_with_noautolint: bool = True -) -> ParseResults: - """Parse all Python files in a directory, collecting successes and failures. - - Args: - directory: Directory to search for Python files. - skip_subdirs_with_noautolint: If True, skip subdirectories that contain - a .noautolint file. The root eval directory's .noautolint is not - affected by this (it uses per-check suppressions). - - Returns: - ParseResults with all parse attempts, both successful and failed. - """ - results = ParseResults() - - # Find subdirectories with .noautolint files to skip entirely - # (excluding the root eval directory itself) - skip_dirs: set[Path] = set() - if skip_subdirs_with_noautolint: - all_noautolint_dirs = _get_noautolint_directories(directory) - # Exclude the root directory - its .noautolint uses per-check suppressions - skip_dirs = {d for d in all_noautolint_dirs if d != directory} - - for py_file in directory.rglob("*.py"): - # Skip files in directories with their own .noautolint - if skip_dirs and _is_path_under_any(py_file, skip_dirs): - continue - results.results.append(safe_parse_file(py_file)) - - return results - - -def add_parse_errors_to_report( - check_name: str, - failed_paths: list[str], - report: LintReport, -) -> bool: - """Add a failure result if there are parse errors. - - Args: - check_name: Name of the check that encountered parse errors. - failed_paths: List of file paths that failed to parse. - report: LintReport to add the failure to. - - Returns: - True if there were parse errors (caller should return early), False otherwise. - """ - if not failed_paths: - return False - - from tools.run_autolint.models import LintResult - - report.add( - LintResult( - name=check_name, - status="fail", - message=f"Could not parse {len(failed_paths)} file(s): {failed_paths}", - ) - ) - return True diff --git a/tools/run_autolint/models.py b/tools/run_autolint/models.py deleted file mode 100644 index f9c3842..0000000 --- a/tools/run_autolint/models.py +++ /dev/null @@ -1,89 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Data models for autolint.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Literal - - -@dataclass -class LintResult: - """Result of a single lint check.""" - - name: str - status: Literal["pass", "fail", "warn", "skip", "suppressed"] - message: str - file: str | None = None - line: int | None = None - - -@dataclass -class LintReport: - """Collection of lint results for an evaluation.""" - - eval_name: str - results: list[LintResult] = field(default_factory=list) - - def add(self, result: LintResult) -> None: - self.results.append(result) - - def passed(self) -> bool: - return all( - r.status in ("pass", "warn", "skip", "suppressed") for r in self.results - ) - - def summary(self) -> dict[str, int]: - counts: dict[str, int] = { - "pass": 0, - "fail": 0, - "warn": 0, - "skip": 0, - "suppressed": 0, - } - for r in self.results: - counts[r.status] += 1 - return counts - - -@dataclass -class Suppressions: - """Track suppressions for an evaluation.""" - - eval_level: set[str] = field(default_factory=set) - dir_level: dict[str, set[str]] = field(default_factory=dict) - file_level: dict[str, set[str]] = field(default_factory=dict) - line_level: dict[str, dict[int, set[str]]] = field(default_factory=dict) - - def is_suppressed( - self, check_name: str, file_path: str | None = None, line: int | None = None - ) -> bool: - """Check if a given check is suppressed.""" - # Eval-level suppression - if check_name in self.eval_level: - return True - - if file_path: - # Directory-level suppression - for dir_path, checks in self.dir_level.items(): - if ( - Path(file_path).is_relative_to(Path(dir_path)) - and check_name in checks - ): - return True - - # File-level suppression - if ( - file_path in self.file_level - and check_name in self.file_level[file_path] - ): - return True - - # Line-level suppression - if line and file_path in self.line_level: - if line in self.line_level[file_path]: - if check_name in self.line_level[file_path][line]: - return True - - return False diff --git a/tools/run_autolint/output.py b/tools/run_autolint/output.py deleted file mode 100644 index 2a284ab..0000000 --- a/tools/run_autolint/output.py +++ /dev/null @@ -1,245 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Rich-based output for autolint.""" - -from __future__ import annotations - -from rich.console import Console -from rich.rule import Rule -from rich.table import Table -from rich.text import Text -from tools.run_autolint.models import LintReport - -console = Console() - - -def get_status_text(status: str) -> Text: - """Get styled status indicator.""" - status_styles = { - "pass": "[bold green]PASS[/]", - "fail": "[bold red]FAIL[/]", - "warn": "[bold yellow]WARN[/]", - "skip": "[dim]SKIP[/]", - "suppressed": "[bold blue]SUPP[/]", - } - markup = status_styles.get(status, status.upper()) - return Text.from_markup(markup) - - -def print_report(report: LintReport) -> None: - """Print the lint report using rich.""" - console.print() - console.print(Rule(f"[bold]Lint Report: {report.eval_name}[/]", style="blue")) - console.print() - - # Create results table - table = Table(show_header=True, header_style="bold", box=None, padding=(0, 1)) - table.add_column("Status", width=6) - table.add_column("Check", style="cyan") - table.add_column("Message", overflow="fold") - table.add_column("Location", style="dim", overflow="fold") - - failed_checks: list[str] = [] - - for result in report.results: - status_text = get_status_text(result.status) - - # Build location string - location = "" - if result.file: - location = result.file - if result.line: - location += f":{result.line}" - - # Style message based on status - message_style = { - "pass": "", - "fail": "red", - "warn": "yellow", - "skip": "dim", - "suppressed": "blue", - }.get(result.status, "") - - table.add_row( - status_text, - result.name, - Text(result.message, style=message_style), - location, - ) - - if result.status == "fail": - failed_checks.append(result.name) - - console.print(table) - - # Print summary - summary = report.summary() - console.print() - console.print(Rule(style="dim")) - - summary_parts = [] - if summary["pass"] > 0: - summary_parts.append(f"[green]{summary['pass']} passed[/]") - if summary["fail"] > 0: - summary_parts.append(f"[red]{summary['fail']} failed[/]") - if summary["warn"] > 0: - summary_parts.append(f"[yellow]{summary['warn']} warnings[/]") - if summary["skip"] > 0: - summary_parts.append(f"[dim]{summary['skip']} skipped[/]") - if summary["suppressed"] > 0: - summary_parts.append(f"[blue]{summary['suppressed']} suppressed[/]") - - console.print(f"Summary: {', '.join(summary_parts)}") - - if report.passed(): - console.print("[bold green]All required checks passed![/]") - else: - console.print("[bold red]Some checks failed.[/]") - console.print() - console.print("[dim]To suppress a check, add one of:[/]") - for check in failed_checks: - console.print(f" [cyan]# noautolint: {check}[/] [dim](on the line)[/]") - console.print( - f" [dim]Or add check name to[/] [cyan]src/{report.eval_name}/.noautolint[/] [dim](eval-level)[/]" - ) - console.print( - f" [dim]Or add check name to[/] [cyan]src/{report.eval_name}//.noautolint[/] [dim](dir-level)[/]" - ) - console.print() - console.print("[dim]More info about these checks is in AUTOMATED_CHECKS.md") - - -_PASS_RATE_HIGH = 0.80 -_PASS_RATE_MED = 0.50 - - -def _pass_rate_text(rate: float) -> Text: - """Return a styled Text for a pass rate (0.0–1.0).""" - pct = rate * 100 - if rate == 1.0: - return Text("100%", style="bold green") - if rate >= _PASS_RATE_HIGH: - return Text(f"{pct:.0f}%", style="green") - if rate >= _PASS_RATE_MED: - return Text(f"{pct:.0f}%", style="yellow") - return Text(f"{pct:.0f}%", style="red") - - -def print_check_summary(reports: list[LintReport]) -> None: - """Print per-check compliance summary across all evaluations.""" - from collections import defaultdict - - # Aggregate results per check name - check_counts: dict[str, dict[str, int]] = defaultdict( - lambda: {"pass": 0, "fail": 0, "warn": 0, "skip": 0, "suppressed": 0} - ) - for report in reports: - for result in report.results: - check_counts[result.name][result.status] += 1 - - console.print() - console.print(Rule("[bold]CHECK COMPLIANCE SUMMARY[/]", style="blue")) - console.print() - - table = Table(show_header=True, header_style="bold", box=None, padding=(0, 1)) - table.add_column("Check", style="cyan") - table.add_column("Pass", justify="right", style="green") - table.add_column("Fail", justify="right", style="red") - table.add_column("Warn", justify="right", style="yellow") - table.add_column("Skip", justify="right", style="dim") - table.add_column("Supp", justify="right", style="blue") - table.add_column("% Pass", justify="right") - - # Sort by pass rate ascending (worst compliance first) - sorted_checks = sorted( - check_counts.items(), - key=lambda item: (item[1]["pass"] / max(item[1]["pass"] + item[1]["fail"], 1)), - ) - - for check_name, counts in sorted_checks: - applicable = counts["pass"] + counts["fail"] + counts["warn"] - if applicable > 0: - pass_pct = counts["pass"] / applicable - rate_text = _pass_rate_text(pass_pct) - else: - rate_text = Text("-", style="dim") - - table.add_row( - check_name, - str(counts["pass"]), - str(counts["fail"]), - str(counts["warn"]), - str(counts["skip"]), - str(counts["suppressed"]), - rate_text, - ) - - console.print(table) - console.print() - - -def print_overall_summary(reports: list[LintReport]) -> None: - """Print overall summary for multiple evaluations.""" - console.print() - console.print(Rule("[bold]OVERALL SUMMARY[/]", style="blue")) - console.print() - - # Create summary table - table = Table(show_header=True, header_style="bold", box=None, padding=(0, 1)) - table.add_column("Status", width=6) - table.add_column("Evaluation", style="cyan") - table.add_column("Pass", justify="right", style="green") - table.add_column("Fail", justify="right", style="red") - table.add_column("Warn", justify="right", style="yellow") - table.add_column("Skip", justify="right", style="dim") - - passed = 0 - total_pass = 0 - total_fail = 0 - total_warn = 0 - total_skip = 0 - - for report in reports: - summary = report.summary() - total_pass += summary["pass"] - total_fail += summary["fail"] - total_warn += summary["warn"] - total_skip += summary["skip"] - - if report.passed(): - status = Text.from_markup("[bold green]PASS[/]") - passed += 1 - else: - status = Text.from_markup("[bold red]FAIL[/]") - - table.add_row( - status, - report.eval_name, - str(summary["pass"]), - str(summary["fail"]), - str(summary["warn"]), - str(summary["skip"]), - ) - - console.print(table) - console.print() - - # Print totals - console.print(Rule(style="dim")) - console.print( - f"[bold]Totals:[/] " - f"[green]{total_pass} passed[/], " - f"[red]{total_fail} failed[/], " - f"[yellow]{total_warn} warnings[/], " - f"[dim]{total_skip} skipped[/]" - ) - console.print() - - failed = len(reports) - passed - if failed == 0: - console.print( - f"[bold green]{passed}/{len(reports)} evaluations passed all required checks[/]" - ) - else: - console.print( - f"[bold]{passed}/{len(reports)} evaluations passed[/], [red]{failed} failed[/]" - ) diff --git a/tools/run_autolint/suppressions.py b/tools/run_autolint/suppressions.py deleted file mode 100644 index 4d7fffc..0000000 --- a/tools/run_autolint/suppressions.py +++ /dev/null @@ -1,77 +0,0 @@ -# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md -"""Suppression loading for autolint.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from tools.run_autolint.models import LintResult, Suppressions - -# Suppression comment patterns -NOAUTOLINT_LINE_PATTERN = re.compile(r"#\s*noautolint:\s*([\w,\s]+)") -NOAUTOLINT_FILE_PATTERN = re.compile(r"#\s*noautolint-file:\s*([\w,\s]+)") - - -def load_suppressions(eval_path: Path) -> Suppressions: - """Load suppressions from .noautolint file and scan source files.""" - suppressions = Suppressions() - - # Load eval-level and directory-level suppressions from .noautolint files - for noautolint_file in eval_path.rglob(".noautolint"): - dir_suppressed_checks = set() - for raw_line in noautolint_file.read_text().splitlines(): - stripped_line = raw_line.strip() - if stripped_line and not stripped_line.startswith("#"): - dir_suppressed_checks.add(stripped_line) - - if noautolint_file.parent == eval_path: - suppressions.eval_level.update(dir_suppressed_checks) - else: - suppressions.dir_level[str(noautolint_file.parent)] = dir_suppressed_checks - - # Scan Python files for file-level and line-level suppressions - for py_file in eval_path.rglob("*.py"): - try: - content = py_file.read_text() - lines = content.splitlines() - except (OSError, UnicodeDecodeError): - continue - - file_key = str(py_file) - max_file_header_lines = 10 - - for i, source_line in enumerate(lines, start=1): - # Check for file-level suppression (must be in first 10 lines) - if i <= max_file_header_lines: - file_match = NOAUTOLINT_FILE_PATTERN.search(source_line) - if file_match: - file_suppressed_checks = { - c.strip() for c in file_match.group(1).split(",") - } - if file_key not in suppressions.file_level: - suppressions.file_level[file_key] = set() - suppressions.file_level[file_key].update(file_suppressed_checks) - - # Check for line-level suppression - line_match = NOAUTOLINT_LINE_PATTERN.search(source_line) - if line_match: - line_suppressed_checks = { - c.strip() for c in line_match.group(1).split(",") - } - if file_key not in suppressions.line_level: - suppressions.line_level[file_key] = {} - if i not in suppressions.line_level[file_key]: - suppressions.line_level[file_key][i] = set() - suppressions.line_level[file_key][i].update(line_suppressed_checks) - - return suppressions - - -def apply_suppressions(results: list[LintResult], suppressions: Suppressions) -> None: - """Apply suppressions to results, changing fail/warn to suppressed where applicable.""" - for result in results: - if result.status in ("fail", "warn"): - if suppressions.is_suppressed(result.name, result.file, result.line): - result.status = "suppressed" - result.message = f"[suppressed] {result.message}" diff --git a/tools/run_checks.sh b/tools/run_checks.sh index 272f18d..541d313 100755 --- a/tools/run_checks.sh +++ b/tools/run_checks.sh @@ -99,7 +99,7 @@ run_check LARGE_FILES "Large-file scan" bash -c ' exit 1 fi ' -run_check AUTOLINT "Autolint" uv run python tools/run_autolint.py --all-evals +run_check AUTOLINT "Autolint (inspect-evals-lint)" uv run inspect-evals-lint --all echo "${BOLD}════════════════════ Summary ════════════════════${RESET}" echo "Passed: ${#PASSED[@]}" diff --git a/uv.lock b/uv.lock index add5faa..cb714c1 100644 --- a/uv.lock +++ b/uv.lock @@ -1427,6 +1427,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/cc/93f35bc14c6036f774dab0d6c379cc3a72efd9153e19ac7c6b9f88f5cf99/inspect_ai-0.3.235-py3-none-any.whl", hash = "sha256:3690a20dc4d7fe0f52b395d0a4d2286749195a27f03350319a66d6116ececbc4", size = 36437159, upload-time = "2026-06-03T22:55:08.961Z" }, ] +[[package]] +name = "inspect-evals-lint" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/b3/38aa64249bf862aaf3ce9aa4af0e6b60537d4876c8e50721829755d512a8/inspect_evals_lint-0.4.2.tar.gz", hash = "sha256:fc16557369d5dd8ede0efbf25631812692924db7af31513914799771c1dac316", size = 193965, upload-time = "2026-09-21T08:50:54.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/87/b7555007feaf431031b9a7b172826f37eaa1146484de0622fec8df0414ad/inspect_evals_lint-0.4.2-py3-none-any.whl", hash = "sha256:41076b18142a866ce1bb27cf5d83bd451b1949527085d699c26c871721390f05", size = 76684, upload-time = "2026-09-21T08:50:53.072Z" }, +] + [[package]] name = "inspect-scout" version = "0.4.27" @@ -1797,11 +1810,14 @@ dependencies = [ [package.optional-dependencies] lab-bench-2 = [ { name = "labbench2" }, + { name = "pillow" }, + { name = "pydantic" }, ] [package.dev-dependencies] dev = [ { name = "anthropic" }, + { name = "inspect-evals-lint" }, { name = "inspect-scout" }, { name = "mypy" }, { name = "openai" }, @@ -1821,6 +1837,8 @@ requires-dist = [ { name = "datasets", specifier = ">=4.8.5" }, { name = "inspect-ai", specifier = ">=0.3.234" }, { name = "labbench2", marker = "extra == 'lab-bench-2'", git = "https://github.com/EdisonScientific/labbench2.git?rev=c028ecdcf144b55ffcd92b68be45081df5628c20" }, + { name = "pillow", marker = "extra == 'lab-bench-2'" }, + { name = "pydantic", marker = "extra == 'lab-bench-2'" }, { name = "pyyaml", specifier = ">=5.1.0" }, ] provides-extras = ["lab-bench-2", "test"] @@ -1828,6 +1846,7 @@ provides-extras = ["lab-bench-2", "test"] [package.metadata.requires-dev] dev = [ { name = "anthropic", specifier = ">=0.105.0" }, + { name = "inspect-evals-lint", specifier = "==0.4.2" }, { name = "inspect-scout", specifier = ">=0.3.0" }, { name = "mypy" }, { name = "openai", specifier = ">=2.40.0" }, From a129f4ca0be9ec6f01b8e3f76321176ba085f20c Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 21 Sep 2026 18:52:20 +1000 Subject: [PATCH 2/2] Replace .noautolint with a tests package Neither entry needs carrying over. The linter now treats the repository's own src/utils as first-party, and tests/__init__.py makes the test package tests.lab_bench_2, which no longer shadows src/lab_bench_2, so the test directories can have __init__.py files. Co-Authored-By: Claude Fable 5.1 --- src/lab_bench_2/.noautolint | 12 ------------ tests/__init__.py | 0 tests/lab_bench_2/__init__.py | 0 tests/lab_bench_2/solvers/__init__.py | 0 4 files changed, 12 deletions(-) delete mode 100644 src/lab_bench_2/.noautolint create mode 100644 tests/__init__.py create mode 100644 tests/lab_bench_2/__init__.py create mode 100644 tests/lab_bench_2/solvers/__init__.py diff --git a/src/lab_bench_2/.noautolint b/src/lab_bench_2/.noautolint deleted file mode 100644 index a25cf01..0000000 --- a/src/lab_bench_2/.noautolint +++ /dev/null @@ -1,12 +0,0 @@ -# external_dependencies: false positive. `utils` is the repo's first-party helper -# package (src/utils, same distribution), used via the documented -# load_version_from_yaml pattern (see src/examples/gpqa). It is not an external -# dependency, but the check resolves it to the installed distribution and flags it. -external_dependencies - -# tests_init: inapplicable to this template's layout. Eval packages live at the top -# level (src/lab_bench_2), so a tests/lab_bench_2/__init__.py makes pytest's prepend -# importer treat the test directory as the `lab_bench_2` package and shadow the -# source package, breaking collection. The template's own example evals -# (tests/examples/*) likewise omit __init__.py. -tests_init diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/lab_bench_2/__init__.py b/tests/lab_bench_2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/lab_bench_2/solvers/__init__.py b/tests/lab_bench_2/solvers/__init__.py new file mode 100644 index 0000000..e69de29