Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
345 changes: 345 additions & 0 deletions pnpm-lock.yaml

Large diffs are not rendered by default.

21 changes: 21 additions & 0 deletions website/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,24 @@ model and treatment. Each judge shows its score, scoring criteria, rationale,
and file-backed findings with code snippets. Scores use the judge's configured
scale, not a shared pass/fail threshold. Judge errors and missing results are
shown separately from scored results.

Each trial's **Code** tab shows the saved `artifacts.workspaceDirectory` as an
expandable file tree with read-only UTF-8 text previews. This is the final saved
workspace, including starter files, rather than a diff of the agent's changes.
Workspaces are read at build time, so the explorer also works in the static export.
Missing workspaces have an unavailable message.

Recognized file types use Shiki syntax highlighting with GitHub light and dark
themes that follow the website's color mode. Unknown file types remain plain text.
Previews are highlighted on the server and exported as individual JSON assets.
The explorer fetches a preview only when its file is selected, keeping file contents,
tokens, Shiki, and language grammars out of the initial results page. Highlighting
runs at build time for the static export, or on request during local development.
The browser renders the returned tokens as escaped text, not generated HTML.

Dependency, build, and Git directories (`node_modules`, `.next`, `.turbo`, `dist`,
and `.git`) are omitted. Symbolic links and binary files cannot be previewed.
Previews are limited to 256 KiB per file and 2 MiB per workspace; the tree is limited
to 2,000 entries and 50 directory levels. Limits are indicated in the explorer.
Review workspace contents before publishing a result bundle, as previewable files
are included in the website.
4 changes: 3 additions & 1 deletion website/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
"@primer/tailwind-config": "^1.0.0",
"next": "^16.3.1",
"react": "19.2.8",
"react-dom": "19.2.8"
"react-dom": "19.2.8",
"server-only": "^0.0.1",
"shiki": "^4.4.3"
},
"dependenciesMeta": {
"@primer/agent-eval": {
Expand Down
72 changes: 72 additions & 0 deletions website/src/app/components/FileExplorer.module.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
.explorer {
display: grid;
grid-template-columns: minmax(180px, 280px) minmax(0, 1fr);
border: 1px solid var(--borderColor-default);
border-radius: var(--borderRadius-medium);
overflow: hidden;
}

.tree {
max-height: 600px;
overflow: auto;
padding: var(--base-size-8);
border-right: 1px solid var(--borderColor-default);
}

.preview {
min-width: 0;
}

.header {
display: flex;
align-items: baseline;
justify-content: space-between;
gap: var(--base-size-16);
padding: var(--base-size-12);
background: var(--bgColor-muted);
border-bottom: 1px solid var(--borderColor-default);
}

.code {
margin: 0;
padding: var(--base-size-16);
max-height: 550px;
overflow: auto;
font-size: var(--text-body-size-medium);
tab-size: 2;
}

.code span {
color: var(--shiki-light);
font-style: var(--shiki-light-font-style);
font-weight: var(--shiki-light-font-weight);
text-decoration: var(--shiki-light-text-decoration);
}

:global([data-color-mode='dark']) .code span {
color: var(--shiki-dark);
font-style: var(--shiki-dark-font-style);
font-weight: var(--shiki-dark-font-weight);
text-decoration: var(--shiki-dark-text-decoration);
}

@media (prefers-color-scheme: dark) {
:global([data-color-mode='auto']) .code span {
color: var(--shiki-dark);
font-style: var(--shiki-dark-font-style);
font-weight: var(--shiki-dark-font-weight);
text-decoration: var(--shiki-dark-text-decoration);
}
}

@media (max-width: 767px) {
.explorer {
grid-template-columns: minmax(0, 1fr);
}

.tree {
max-height: 240px;
border-right: 0;
border-bottom: 1px solid var(--borderColor-default);
}
}
82 changes: 82 additions & 0 deletions website/src/app/components/FileExplorer.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
'use client'

import {FileIcon} from '@primer/octicons-react'
import {TreeView} from '@primer/react'
import {useId, useState} from 'react'
import type {FilePreviewReference} from '../../file-preview'
import type {WorkspaceEntry, WorkspaceFile, WorkspaceFiles} from '../../workspace-files'
import {FilePreview} from './FilePreview'
import styles from './FileExplorer.module.css'

function FileExplorer({workspace}: {workspace: WorkspaceFiles<FilePreviewReference>}) {
const [selectedFile, setSelectedFile] = useState<WorkspaceFile<FilePreviewReference> | null>(null)
const id = useId()

if (workspace.type === 'unavailable') {
return <p className="m-0 text-muted">{workspace.reason}</p>
}

function renderEntry(entry: WorkspaceEntry<FilePreviewReference>) {
if (entry.type === 'directory') {
return (
<TreeView.Item id={`${id}-${entry.path}`} key={entry.path}>
<TreeView.LeadingVisual>
<TreeView.DirectoryIcon />
</TreeView.LeadingVisual>
{entry.name}
<TreeView.SubTree>{entry.children.map(renderEntry)}</TreeView.SubTree>
</TreeView.Item>
)
}

return (
<TreeView.Item
current={selectedFile?.path === entry.path}
id={`${id}-${entry.path}`}
key={entry.path}
onSelect={() => {
setSelectedFile(entry)
}}
>
<TreeView.LeadingVisual>
<FileIcon />
</TreeView.LeadingVisual>
{entry.name}
</TreeView.Item>
)
}

if (workspace.entries.length === 0) {
return <p className="m-0 text-muted">No files are available in the generated workspace.</p>
}

return (
<div>
{workspace.truncated ? (
<p className="mt-0 text-muted">The file tree is limited to 2,000 entries and 50 directory levels.</p>
) : null}
<div className={styles.explorer}>
<div className={styles.tree}>
<TreeView aria-label="Generated workspace files">{workspace.entries.map(renderEntry)}</TreeView>
</div>
<section aria-label="File preview" className={styles.preview}>
{selectedFile ? (
<>
<header className={styles.header}>
<h3 className="text-body-medium m-0 break-all">{selectedFile.path}</h3>
<span className="text-caption text-muted whitespace-nowrap">
{selectedFile.size.toLocaleString('en-US')} bytes
</span>
</header>
<FilePreview file={selectedFile} key={selectedFile.path} />
</>
) : (
<p className="p-3 m-0 text-muted">Select a file to view its contents.</p>
)}
</section>
</div>
</div>
)
}

export {FileExplorer}
127 changes: 127 additions & 0 deletions website/src/app/components/FilePreview.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import {renderToStaticMarkup} from 'react-dom/server'
import {expect, test, vi} from 'vitest'
import {getFileLanguage, highlightFile} from '../../file-highlighting'
import {isFilePreviewData} from '../../file-preview'
import type {WorkspaceFile} from '../../workspace-files'
import {FilePreview, FilePreviewContent} from './FilePreview'

vi.mock('server-only', () => {
return {}
})
vi.mock('@primer/react', () => {
return {Button: 'button'}
})

function file(filepath: string, content: string): WorkspaceFile {
return {
type: 'file',
path: filepath,
name: filepath.split('/').at(-1) ?? filepath,
size: Buffer.byteLength(content),
preview: {type: 'text', content},
}
}

async function renderFile(workspaceFile: WorkspaceFile) {
const preview = await highlightFile(workspaceFile)
expect(isFilePreviewData(JSON.parse(JSON.stringify(preview)))).toBe(true)
return renderToStaticMarkup(<FilePreviewContent filepath={workspaceFile.path} preview={preview} />)
}

test.each([
['src/App.tsx', 'tsx'],
['src/index.ts', 'typescript'],
['src/types.d.ts', 'typescript'],
['src/index.mts', 'typescript'],
['src/App.jsx', 'jsx'],
['next.config.mjs', 'javascript'],
['index.cjs', 'javascript'],
['package.json', 'json'],
['styles.CSS', 'css'],
['README.md', 'markdown'],
['workflow.yml', 'yaml'],
['script.py', 'python'],
['script.sh', 'shellscript'],
['Dockerfile', 'docker'],
['Dockerfile.dev', 'docker'],
['Makefile', 'make'],
['.env.local', 'dotenv'],
['LICENSE', 'text'],
['file.unknown', 'text'],
])('detects the language for %s', (filepath, language) => {
expect(getFileLanguage(filepath)).toBe(language)
})

test.each([
['src/App.tsx', 'export default function App() {\n return <button>Hello</button>\n}\n'],
['package.json', '{\n "name": "generated-app"\n}\n'],
['styles.css', 'button {\n color: red;\n}\n'],
])('renders server-highlighted %s tokens with both themes', async (filepath, content) => {
const html = await renderFile(file(filepath, content))
expect(html).toContain('--shiki-light:')
expect(html).toContain('--shiki-dark:')
expect(html).toContain(`aria-label="${filepath}"`)
expect(html).toContain('tabindex="0"')
expect(html.replace(/<[^>]*>/g, '')).toBe(renderToStaticMarkup(<>{content}</>))
})

test.each([
'\n\n\tconst greeting = "hello"\n\n',
'\r\n\tconst greeting = "hello"\r\n\r\n',
'\nconst first = 1\r\nconst second = 2\n',
'const greeting = "hello"',
'const greeting = "こんにちは 👋"\n',
])('preserves source text, whitespace, and line endings: %j', async content => {
const html = await renderFile(file('index.ts', content))
expect(html.replace(/<[^>]*>/g, '')).toBe(renderToStaticMarkup(<>{content}</>))
})

test('escapes file contents rather than rendering generated HTML', async () => {
const content = '<script>alert("generated")</script><img src=x onerror=alert(1)>'
const html = await renderFile(file('index.html', content))
expect(html).not.toContain('<script>')
expect(html).not.toContain('<img ')
expect(html.replace(/<[^>]*>/g, '')).toBe(renderToStaticMarkup(<>{content}</>))
})

test('renders unknown file types as escaped plain text', async () => {
const content = '<script>alert("generated")</script>\nplain text\n'
const html = await renderFile(file('notes.unknown', content))
expect(html).not.toContain('<span')
expect(html).not.toContain('--shiki-')
expect(html).toContain('&lt;script&gt;')
expect(html.replace(/<[^>]*>/g, '')).toBe(renderToStaticMarkup(<>{content}</>))
})

test('preserves empty and unavailable preview messages', async () => {
const empty = await renderFile(file('empty.ts', ''))
expect(empty).toContain('This file is empty.')
const unavailable = await renderFile({
...file('binary.png', ''),
preview: {type: 'unavailable', reason: 'Binary files cannot be previewed.'},
})
expect(unavailable).toContain('Binary files cannot be previewed.')
expect(unavailable).not.toContain('<pre')
})

test('renders only a loading state before fetching a selected remote preview', () => {
const html = renderToStaticMarkup(
<FilePreview
file={{...file('index.ts', ''), preview: {type: 'remote', url: '/file-previews/example/preview.json'}}}
/>,
)
expect(html).toContain('Loading file preview...')
expect(html).not.toContain('<pre')
})

test.each([
null,
{},
{type: 'text', content: 1},
{type: 'unavailable'},
{type: 'highlighted', content: '', tokens: null},
{type: 'highlighted', content: '', tokens: [{content: 'x', offset: -1, style: {}}]},
{type: 'highlighted', content: '', tokens: [{content: 'x', offset: 0, style: {'--shiki-light': 1}}]},
])('rejects malformed preview responses: %j', value => {
expect(isFilePreviewData(value)).toBe(false)
})
Loading