Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 43 additions & 12 deletions apps/web/astro.config.mjs
Original file line number Diff line number Diff line change
@@ -1,27 +1,58 @@
import { readFileSync } from 'node:fs';
import { readFileSync, readdirSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import starlight from '@astrojs/starlight';
import { defineConfig } from 'astro/config';

// Static builds can't redirect an open-ended `/docs/[...slug]` wildcard to
// v4.42.4 (that requires enumerable paths), so generate one concrete
// redirect per known v4.42.4 route from its route manifest instead.
const v4RoutesPath = fileURLToPath(new URL('./src/data/docs-v4.42.4-routes.json', import.meta.url));
const v4Routes = JSON.parse(readFileSync(v4RoutesPath, 'utf8'));
const v4Redirects = Object.fromEntries(
v4Routes.map((route) => {
const bareRoute = route.replace('/docs/v4.42.4/', '/docs/');
const from = bareRoute === '/docs/' ? '/docs' : bareRoute.replace(/\/$/, '');
return [from, route];
const currentDocsRoot = fileURLToPath(new URL('./src/content/docs/docs/next', import.meta.url));

const currentRoutes = collectDocsRoutes(currentDocsRoot, '/docs');
const nextRedirects = Object.fromEntries(
currentRoutes.map((route) => {
const suffix = route === '/docs/' ? '' : route.slice('/docs/'.length);
const from = suffix ? `/docs/next/${suffix}` : '/docs/next';
return [from.replace(/\/$/, ''), route];
}),
);

function collectDocsRoutes(root, base) {
return collectMarkdownFiles(root)
.map((file) => {
const slug = getFrontmatterSlug(file);
if (slug) return `/${slug.replace(/^\/|\/$/g, '')}/`;

const relative = path.relative(root, file).replace(/\\/g, '/');
const suffix = relative
.replace(/\.mdx?$/, '')
.replace(/(^|\/)index$/, '')
.replace(/\/$/, '');
return suffix ? `${base}/${suffix}/` : `${base}/`;
})
.sort();
}

function getFrontmatterSlug(file) {
const source = readFileSync(file, 'utf8');
return source
.split('---')[1]
?.match(/^slug:\s*(.+)$/m)?.[1]
?.trim();
}

function collectMarkdownFiles(dir) {
return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) return collectMarkdownFiles(fullPath);
return /\.mdx?$/.test(entry.name) ? [fullPath] : [];
});
}

export default defineConfig({
site: 'https://agentv.dev',
image: { service: { entrypoint: 'astro/assets/services/noop' } },
redirects: {
'/docs/v4': '/docs/v4.42.4/',
...v4Redirects,
...nextRedirects,
},
integrations: [
starlight({
Expand Down
1 change: 1 addition & 0 deletions apps/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"dev": "astro dev",
"docs:snapshot:v4.42.4": "node ../../scripts/snapshot-docs-version.mjs v4.42.4",
"build": "astro build",
"check:routes": "node scripts/check-doc-routes.mjs",
"preview": "astro preview"
},
"dependencies": {
Expand Down
69 changes: 69 additions & 0 deletions apps/web/scripts/check-doc-routes.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const webRoot = fileURLToPath(new URL('..', import.meta.url));
const distRoot = path.join(webRoot, 'dist');
const v4Routes = JSON.parse(
readFileSync(path.join(webRoot, 'src/data/docs-v4.42.4-routes.json'), 'utf8'),
);

const checks = [
{
path: 'docs/index.html',
includes: ['Introduction | AgentV', ' Current '],
excludes: ['Redirecting to:'],
},
{
path: 'docs/targets/llm-providers/index.html',
includes: ['LLM Providers | AgentV', '<h1 id="_top"'],
excludes: ['Redirecting to:'],
},
{
path: 'docs/v4.42.4/targets/llm-providers/index.html',
includes: ['LLM Providers | AgentV', '/docs/v4.42.4/targets/llm-providers/'],
excludes: ['Redirecting to:', '/docs/v4424/'],
},
{
path: 'docs/next/targets/llm-providers/index.html',
includes: ['Redirecting to: /docs/targets/llm-providers/'],
},
];

for (const check of checks) {
const file = path.join(distRoot, check.path);
if (!existsSync(file)) {
fail(`Missing built docs route: ${check.path}`);
}

const html = readFileSync(file, 'utf8');
for (const expected of check.includes ?? []) {
if (!html.includes(expected)) {
fail(`${check.path} does not include expected content: ${expected}`);
}
}

for (const unexpected of check.excludes ?? []) {
if (html.includes(unexpected)) {
fail(`${check.path} includes unexpected content: ${unexpected}`);
}
}
}

if (existsSync(path.join(distRoot, 'docs/v4424'))) {
fail('Unexpected normalized archive route directory emitted: docs/v4424');
}

for (const route of v4Routes) {
const routeFile = path.join(distRoot, route, 'index.html');
if (!existsSync(routeFile)) {
fail(`Missing archived manifest route: ${route}`);
}
}

console.log('Docs route checks passed');

function fail(message) {
console.error(message);
process.exit(1);
}
2 changes: 1 addition & 1 deletion apps/web/src/components/VersionSelect.astro
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
const versions = [
{ label: 'Next', base: '/docs/next' },
{ label: 'Current', base: '/docs' },
{ label: 'v4.42.4', base: '/docs/v4.42.4' },
];

Expand Down
7 changes: 3 additions & 4 deletions apps/web/src/components/VersionedSidebar.astro
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@ import SidebarSublist from '@astrojs/starlight/components/SidebarSublist.astro';
import type { SidebarEntry } from '@astrojs/starlight/utils/routing/types';
import v4Routes from '../data/docs-v4.42.4-routes.json';

// The Starlight sidebar config autogenerates from docs/next/*, so the base
// sidebar's hrefs already point at the live /docs/next/ tree unmodified.
// Only genuinely archived versions below need their hrefs remapped.
const LIVE_PREFIX = '/docs/next/';
// The Starlight sidebar config autogenerates from docs/next/*, but those files
// publish with root /docs/* slugs. Only archived versions need href remapping.
const LIVE_PREFIX = '/docs/';
const ARCHIVED_VERSIONS = [{ slug: 'v4.42.4', routes: v4Routes }];

const { sidebar } = Astro.locals.starlightRoute;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Batch CLI Evaluation
description: Evaluate external tools that process all tests in a single invocation
sidebar:
order: 5
slug: docs/evaluation/batch-cli
---

Batch CLI evaluation handles tools that process multiple inputs at once — bulk classifiers, screening engines, or any runner that reads all tests and outputs results in one pass.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Tests
description: Defining individual tests
sidebar:
order: 2
slug: docs/evaluation/eval-cases
---

Tests are individual test entries within an evaluation file. Each test defines input messages, expected outcomes, and optional grader overrides.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Eval Files
description: YAML and JSONL evaluation file formats
sidebar:
order: 1
slug: docs/evaluation/eval-files
---

Evaluation files define the test cases, graders, workspace lifecycle, and run
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Example Evaluations
description: Complete working examples of eval files for common patterns
sidebar:
order: 6
slug: docs/evaluation/examples
---

This page collects complete eval file examples you can copy and adapt. Each demonstrates a different AgentV pattern.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Experiments
description: Configure how AgentV evals run
sidebar:
order: 2
slug: docs/evaluation/experiments
---

AgentV eval files are the runnable authoring artifact. Use top-level
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/content/docs/docs/next/evaluation/rubrics.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Rubrics
description: Structured evaluation criteria with weights
sidebar:
order: 3
slug: docs/evaluation/rubrics
---

Rubrics are defined with `assert` entries and support binary checklist grading and score-range analytic grading.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Running Evaluations
description: CLI commands for running and managing evaluations
sidebar:
order: 4
slug: docs/evaluation/running-evals
---

## Run an Evaluation
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/content/docs/docs/next/evaluation/sdk.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: TypeScript SDK
description: Programmatic API for evaluations, custom assertions, and typed configuration
sidebar:
order: 6
slug: docs/evaluation/sdk
---

YAML remains AgentV's canonical, portable eval format. The SDK surfaces below are for cases where you want to generate YAML-shaped definitions in code, embed eval runs inside another application, or write executable graders and prompt templates. For authoring helpers, `@agentv/sdk` is AgentV's public lightweight SDK package.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Installation
description: Install AgentV CLI and get started with bundled skills
sidebar:
order: 2
slug: docs/getting-started/installation
---

## Prerequisites
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Quick Start
description: Create and run your first evaluation
sidebar:
order: 3
slug: docs/getting-started/quickstart
---

Follow these steps to create and run your first evaluation.
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/content/docs/docs/next/graders/composite.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Composite Graders
description: Combine multiple graders with aggregation strategies for multi-criteria evaluation.
sidebar:
order: 4
slug: docs/graders/composite
---

Composite graders combine multiple graders and aggregate their results into a single score. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Custom Assertions
description: Build reusable assertion types with defineAssertion() and convention-based discovery
sidebar:
order: 7
slug: docs/graders/custom-assertions
---

Custom assertions let you add evaluation logic that goes beyond built-in types. Define a TypeScript function, drop it in `.agentv/assertions/`, and reference it by name in your YAML eval files.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Custom Graders
description: Patterns for building custom evaluation logic
sidebar:
order: 3
slug: docs/graders/custom-graders
---

AgentV supports multiple grader types that can be combined for comprehensive evaluation.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Execution Metrics
description: Threshold-based checks on execution metrics
sidebar:
order: 5
slug: docs/graders/execution-metrics
---

AgentV provides built-in graders for checking execution metrics against thresholds. These are useful for enforcing efficiency constraints without writing custom code.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: LLM Graders
description: Customizable LLM-based evaluation
sidebar:
order: 2
slug: docs/graders/llm-graders
---

LLM graders use a language model to evaluate agent responses against custom criteria defined in a prompt file.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Repo-Local Python Helpers
description: Example-local Python helpers for canonical AgentV script graders and eval authoring
sidebar:
order: 7
slug: docs/graders/python-helpers
---

AgentV's Python surface currently starts as a repo-local helper example, not a separate runner or published package.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Script Graders
description: Deterministic script graders in Python or TypeScript
sidebar:
order: 1
slug: docs/graders/script-graders
---

Script graders are scripts that evaluate agent responses deterministically. Write them in any language — Python, TypeScript, Node, or any executable.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Structured Data & Metrics Graders
description: Built-in graders for JSON field comparison and performance gates (latency, cost, token usage).
sidebar:
order: 6
slug: docs/graders/structured-data
---

Built-in graders for grading structured outputs and gating on execution metrics:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Tool Trajectory Graders
description: Validate that agents use the right tools in the right order with argument matching and latency assertions.
sidebar:
order: 5
slug: docs/graders/tool-trajectory
---

Tool trajectory graders validate that an agent used the expected tools during execution. They work with trace data returned by agent providers (codex, vscode, cli with trace support).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Agent Evaluation Layers
description: A four-layer taxonomy for evaluating AI agents — Reasoning, Action, End-to-End, and Safety — mapped to AgentV graders.
sidebar:
order: 1
slug: docs/guides/agent-eval-layers
---

A practical taxonomy for structuring agent evaluations. Each layer targets a different dimension of agent behavior, and maps directly to AgentV graders you can drop into an `EVAL.yaml`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Autoresearch
description: Run an unattended eval-improve loop that iteratively optimizes agent skills
sidebar:
order: 5
slug: docs/guides/autoresearch
---

import { Image } from 'astro:assets';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Benchmark Provenance
description: Patterns for source pins, task artifacts, hooks, and generated benchmark metadata.
sidebar:
order: 5
slug: docs/guides/benchmark-provenance
---

Benchmark suites usually need more than a prompt and a score. They carry source
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Enterprise Governance
description: A Git-native pattern for inventorying and reviewing the AI systems in your organisation, using a `.ai-register.yaml` per repo and a GitHub Action to aggregate them.
sidebar:
order: 9
slug: docs/guides/enterprise-governance
---

This guide describes a lightweight convention for keeping a documented
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Eval Authoring Guide
description: Practical guidance for writing workspace-based evals that work reliably across providers.
sidebar:
order: 3
slug: docs/guides/eval-authoring
---

## Agent Rules and Skill Paths
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Execution Quality vs Trigger Quality
description: Two distinct evaluation concerns for AI agents and skills — what AgentV measures, and what belongs to skill-creator tooling.
sidebar:
order: 2
slug: docs/guides/evaluation-types
---

Agent evaluation has two fundamentally different concerns: **execution quality** and **trigger quality**. They require different tooling, different methodologies, and different optimization surfaces. Conflating them leads to eval configs that are noisy, hard to maintain, and unreliable.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Skill Improvement Workflow
description: Iteratively evaluate and improve agent skills using AgentV
sidebar:
order: 4
slug: docs/guides/skill-improvement-workflow
---

## Introduction
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Workspace Architecture
description: How AgentV materializes eval workspaces, resolves repo acquisition, and keeps target comparisons fair.
sidebar:
order: 7
slug: docs/guides/workspace-architecture
---

AgentV workspaces are the shared substrate an eval runs against: templates,
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/content/docs/docs/next/index.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ title: Introduction
description: What AgentV is and why it exists
sidebar:
order: 1
slug: docs
---

AgentV is a CLI-first AI agent evaluation framework. It evaluates your agents locally with multi-objective scoring (correctness, latency, cost, safety) from YAML specifications. Deterministic script graders, structured `llm-rubric` scoring, and customizable LLM graders are all version-controlled in Git.
Expand Down
Loading
Loading