Add llm evaluation against human accessibility audits

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ permissions:
 
 jobs:
   checks:
-    name: Dependencies, imports, pipeline
+    name: Dependencies, lint, imports, pipeline
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -53,6 +53,12 @@ jobs:
           print('all entry points import cleanly')
           "
 
+      - name: Evaluator lint
+        run: uv run ruff check vision_aid/evaluation tests
+
+      - name: Network-free test suite
+        run: uv run pytest -q
+
       # Exercises extractors, slicers, prompt templates and the programmatic
       # checkers end to end. --dry-run makes no API calls, so this needs no key
       # and costs nothing.

.gitignore

@@ -212,5 +212,27 @@ output/
 # CI smoke-test output
 ci-output/
 
+# Private model-evaluation inputs and derived artifacts
+/Pristine Accessibility Defect Report.xlsx
+/.model-evaluation/
+/vision_aid/evaluation/human_finding_mappings/
+
+# Node.js / Next.js generated files
+node_modules/
+.next/
+next-env.d.ts
+*.tsbuildinfo
+web/prisma/generated/
+
 # Stray git worktrees — never belong in the index
 .claude/worktrees/
+
+# Local-only issue tracker and detailed evaluator tests
+/.scratch/
+/.local-tests/
+
+# Local architecture decision records
+/docs/adr/
+
+# Local agent context
+CONTEXT.md

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.terminal.useEnvFile": true
+}

AGENTS.md

@@ -0,0 +1,13 @@
+## Agent skills
+
+### Issue tracker
+
+Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`.
+
+### Triage labels
+
+The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`.
+
+### Domain docs
+
+This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`.

CLAUDE.md

@@ -5,6 +5,7 @@
 This project builds LLM-powered tools that analyze websites for WCAG accessibility issues and generate structured remediation reports. It is a Computing for Good course project at Georgia Tech (OMSCS) partnered with the Vision Aid Digital Accessibility Testing Team.
 
 The system has three planned stages:
+
 1. Automated webpage accessibility analysis → structured CSV reports
 2. Code remediation suggestions based on those reports
 3. (Stretch) Chrome extension for on-the-fly accessibility fixes
@@ -116,3 +117,17 @@ Docker image → GHCR (`ghcr.io/c4g/va-dat`) → Coolify at `https://va-dat.c4g.
 - HTML test files can be enormous (500K+ tokens) — do not read them fully into context.
 - Some files may have non-UTF-8 encoding; always use `errors='replace'`.
 - Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-5`, `api_server.py` uses `claude-haiku-4-5-20251001`.
+
+## Agent skills
+
+### Issue tracker
+
+Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`.
+
+### Triage labels
+
+The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`.
+
+### Domain docs
+
+This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`.

entry_points/api_server.py

@@ -19,6 +19,7 @@
 """
 
 import json
+import logging
 import os
 import re
 import shutil
@@ -33,11 +34,12 @@
 PROJECT_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(PROJECT_ROOT))
 
-from entry_points.run_pipeline import run_pipeline  # noqa: E402
-from entry_points.generate_report import generate_report  # noqa: E402
-from vision_aid.ingestion.file_crawler import fetch_page, fetch_pages_nested  # noqa: E402
-from processing_scripts.llm_client.client import is_openai_model, is_gemini_model  # noqa: E402
+from entry_points.generate_report import generate_report
+from entry_points.run_pipeline import run_pipeline
+from processing_scripts.llm_client.client import is_gemini_model, is_openai_model
+from vision_aid.ingestion.file_crawler import fetch_page, fetch_pages_nested
 
+logger = logging.getLogger(__name__)
 
 # ── Multi-page splitting ─────────────────────────────────────────────────────
 
@@ -150,9 +152,14 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non
                     "wcag_criteria": data.get("wcag_criteria", []),
                     "status": status,
                     "error": (api_result or {}).get("error"),
+                    "failure": (api_result or {}).get("failure"),
                     "parsed": parsed,
+                    "provider": (api_result or {}).get("provider"),
+                    "request": (api_result or {}).get("request"),
+                    "usage": usage,
                     "input_tokens": usage.get("input_tokens"),
                     "output_tokens": usage.get("output_tokens"),
+                    "stop_reason": (api_result or {}).get("stop_reason"),
                     "duration_seconds": (api_result or {}).get("duration_seconds"),
                 }
 
@@ -177,8 +184,8 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non
                         "stage": "report_complete",
                         "message": "Report ready",
                     })
-            except Exception as csv_err:
-                print(f"  Warning: CSV generation failed: {csv_err}")
+            except Exception:
+                logger.exception("CSV generation failed")
 
         return {
             "success": True,
@@ -202,6 +209,7 @@ def run_audit(html_content: str, api_key: str, model: str, progress_callback=Non
         }
 
     except Exception as exc:
+        logger.exception("Audit failed")
         return {"success": False, "error": str(exc)}
 
     finally:
@@ -223,7 +231,7 @@ def _try_parse_json(text: str):
 class AuditHandler(BaseHTTPRequestHandler):
     """Handle static file serving and the /api/audit endpoint."""
 
-    def log_message(self, fmt, *args):  # noqa: N802
+    def log_message(self, fmt, *args):
         sys.stderr.write(f"[{self.address_string()}] {fmt % args}\n")
 
     # CORS helpers ─────────────────────────────────────────────────────────────
@@ -233,15 +241,15 @@ def _cors_headers(self):
         self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
         self.send_header("Access-Control-Allow-Headers", "Content-Type")
 
-    def do_OPTIONS(self):  # noqa: N802
+    def do_OPTIONS(self):
         """Handle CORS preflight requests."""
         self.send_response(200)
         self._cors_headers()
         self.end_headers()
 
     # GET ──────────────────────────────────────────────────────────────────────
 
-    def do_GET(self):  # noqa: N802
+    def do_GET(self):
         path = self.path.split("?")[0]
         if path in ("/", "/index.html"):
             self._serve_file(STATIC_DIR / "index.html", "text/html; charset=utf-8")
@@ -264,7 +272,7 @@ def _serve_file(self, file_path: Path, content_type: str):
 
     # POST ─────────────────────────────────────────────────────────────────────
 
-    def do_POST(self):  # noqa: N802
+    def do_POST(self):
         if self.path == "/api/audit":
             self._handle_audit()
         elif self.path == "/api/audit/url":
@@ -520,8 +528,8 @@ def _handle_url_audit(self, nested: bool):
 
     def _handle_validate_key(self):
         """Validate an API key with a lightweight call to the provider."""
-        import urllib.request
         import urllib.error
+        import urllib.request
 
         length = int(self.headers.get("Content-Length", 0))
         body = self.rfile.read(length)
@@ -599,7 +607,7 @@ def main():
     parser.add_argument(
         "--port",
         type=int,
-        default=int(os.getenv("PORT", 8000)),
+        default=int(os.getenv("PORT", "8000")),
         help="Port to listen on (default: $PORT or 8000)",
     )
     parser.add_argument(

entry_points/generate_report.py

@@ -17,11 +17,15 @@
 import argparse
 import csv
 import json
+import logging
 import re
 from dataclasses import dataclass, fields
 from pathlib import Path
+
 from bs4 import BeautifulSoup
 
+logger = logging.getLogger(__name__)
+
 # ── Deduplication prompt ─────────────────────────────────────────────────────
 
 _DEDUP_PROMPT = """\
@@ -280,10 +284,7 @@ def _is_link_fp(row: ReportRow, nav_link_texts: set[str]) -> bool:
         return True
 
     # Links inside <nav> or <footer> are clear in context
-    if link_text.lower() in nav_link_texts:
-        return True
-
-    return False
+    return link_text.lower() in nav_link_texts
 
 
 def _is_decorative_img_fp(row: ReportRow, soup: BeautifulSoup | None) -> bool:
@@ -320,7 +321,7 @@ def _is_decorative_img_fp(row: ReportRow, soup: BeautifulSoup | None) -> bool:
     return False
 
 
-def _is_svg_fp(row: ReportRow, soup: BeautifulSoup | None) -> bool:
+def _is_svg_fp(soup: BeautifulSoup | None) -> bool:
     """Return True if an SVG accessibility finding is a likely false positive.
 
     SVGs inside links or buttons that already have visible text are decorative
@@ -397,7 +398,7 @@ def filter_false_positives(
 
         # Check SVG false positives
         elif "SVG" in category:
-            is_fp = _is_svg_fp(row, soup)
+            is_fp = _is_svg_fp(soup)
 
         if is_fp:
             suppressed.append(row)
@@ -554,7 +555,7 @@ def normalize_programmatic(findings: list[dict], page_title: str,
 
 # ── LLM prompt normalizers ─────────────────────────────────────────────────
 
-def _norm_page_title(data: dict, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_page_title(data: dict, wcag: str) -> list[ReportRow]:
     rows = []
     for issue in data.get("issues", []):
         rows.append(ReportRow(
@@ -571,7 +572,7 @@ def _norm_page_title(data: dict, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_heading_structure(data: dict, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_heading_structure(data: dict, wcag: str) -> list[ReportRow]:
     rows = []
     for issue in data.get("issues", []):
         rows.append(ReportRow(
@@ -600,7 +601,7 @@ def _norm_heading_structure(data: dict, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_link_clarity(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_link_clarity(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         if item.get("is_clear", True):
@@ -620,7 +621,7 @@ def _norm_link_clarity(data: list, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_iframe_titles(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_iframe_titles(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         if item.get("is_descriptive", True):
@@ -640,7 +641,7 @@ def _norm_iframe_titles(data: list, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_landmark_structure(data: dict, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_landmark_structure(data: dict, wcag: str) -> list[ReportRow]:
     rows = []
     for issue in data.get("issues", []):
         rows.append(ReportRow(
@@ -657,7 +658,7 @@ def _norm_landmark_structure(data: dict, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_label_quality(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_label_quality(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         if item.get("is_descriptive", True):
@@ -680,7 +681,7 @@ def _norm_label_quality(data: list, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_required_field_indicators(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_required_field_indicators(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         issues = item.get("issues", [])
@@ -702,7 +703,7 @@ def _norm_required_field_indicators(data: list, wcag: str, **ctx) -> list[Report
     return rows
 
 
-def _norm_informative_alt_quality(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_informative_alt_quality(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         issues = item.get("issues", [])
@@ -724,7 +725,7 @@ def _norm_informative_alt_quality(data: list, wcag: str, **ctx) -> list[ReportRo
     return rows
 
 
-def _norm_decorative_verification(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_decorative_verification(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         if item.get("likely_decorative", True):
@@ -744,7 +745,7 @@ def _norm_decorative_verification(data: list, wcag: str, **ctx) -> list[ReportRo
     return rows
 
 
-def _norm_actionable_image_alt(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_actionable_image_alt(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         issues = item.get("issues", [])
@@ -767,7 +768,7 @@ def _norm_actionable_image_alt(data: list, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_svg_accessibility(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_svg_accessibility(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         issues = item.get("issues", [])
@@ -788,7 +789,7 @@ def _norm_svg_accessibility(data: list, wcag: str, **ctx) -> list[ReportRow]:
     return rows
 
 
-def _norm_icon_font_accessibility(data: list, wcag: str, **ctx) -> list[ReportRow]:
+def _norm_icon_font_accessibility(data: list, wcag: str) -> list[ReportRow]:
     rows = []
     for item in data:
         issues = item.get("issues", [])
@@ -880,6 +881,7 @@ def deduplicate_with_llm(
             response_text = response.choices[0].message.content or "[]"
         else:
             import anthropic
+
             from processing_scripts.llm_client.client import supports_temperature
 
             client = anthropic.Anthropic(api_key=api_key)
@@ -897,7 +899,7 @@ def deduplicate_with_llm(
 
         to_remove: list[int] = safe_parse_json(response_text)
         if not isinstance(to_remove, list):
-            raise ValueError(f"Expected list, got {type(to_remove)}")
+            raise TypeError(f"Expected list, got {type(to_remove)}")
 
         remove_set = {int(i) for i in to_remove}
         original_count = len(rows)
@@ -907,8 +909,8 @@ def deduplicate_with_llm(
             print(f"  LLM deduplication removed: {removed} duplicate(s)")
         return deduped
 
-    except Exception as exc:
-        print(f"  WARNING: LLM deduplication failed ({exc}), skipping.")
+    except Exception:
+        logger.exception("LLM deduplication failed, skipping.")
         return rows
 
 
@@ -1086,4 +1088,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()