Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions scripts/gallery-audit/copilot.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,16 @@ export function stripJsonFence(value) {
return match ? match[1].trim() : trimmed;
}

export function extractJsonObject(value) {
const stripped = stripJsonFence(stripVTControlCharacters(value));
const start = stripped.indexOf('{');
const end = stripped.lastIndexOf('}');
if (start === -1 || end < start) return stripped;
return stripped.slice(start, end + 1);
Comment thread
jaydestro marked this conversation as resolved.
}

function normalizeCopilotJson(value) {
return jsonrepair(stripJsonFence(stripVTControlCharacters(value)));
return jsonrepair(extractJsonObject(value));
}

function exactKeys(value, expected) {
Expand Down Expand Up @@ -129,8 +137,8 @@ export function buildClassificationPrompt({ prompt, candidatePath, auditPath, ca
}

export function buildCopilotArguments({ prompt, candidatePath, auditPath, catalogPath, existingEntries }) {
buildClassificationPrompt({ prompt, candidatePath, auditPath, catalogPath, existingEntries });
return [
'-p', buildClassificationPrompt({ prompt, candidatePath, auditPath, catalogPath, existingEntries }),
'--agent=gallery-curator',
'--silent',
'--stream=off',
Expand All @@ -147,6 +155,7 @@ function invokeCopilot(options) {
return spawnSync('copilot', buildCopilotArguments(options), {
encoding: 'utf8',
env: process.env,
input: buildClassificationPrompt(options),
maxBuffer: 2 * 1024 * 1024,
windowsHide: true,
});
Expand All @@ -160,6 +169,8 @@ export function runCopilotClassification(options) {
for (let attempt = 1; attempt <= 2; attempt += 1) {
try {
const result = execute(options);
if (result.error) throw result.error;
if (result.signal) throw new Error(`Copilot terminated by signal ${result.signal}`);
if (result.status !== 0) throw new Error(`Copilot exited with status ${result.status}: ${(result.stderr ?? '').trim()}`);
const parsed = JSON.parse(normalizeCopilotJson(result.stdout ?? ''));
return { status: 'complete', attempts: attempt, classification: validateClassification(parsed, options.candidates, existingEntries) };
Expand Down
3 changes: 2 additions & 1 deletion scripts/gallery-audit/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ async function classifyOnly({ promote = false } = {}) {
metadata.copilot = { status: 'incomplete', attempts: result.attempts, error: result.error };
}
await writeReports(auditReport, candidateReport, metadata);
if (promote && result.status === 'complete') await applyCatalogPromotion({ root });
if (result.status !== 'complete') throw new Error(`Copilot classification incomplete after ${result.attempts} attempts: ${result.error}`);
if (promote) await applyCatalogPromotion({ root });
}

async function run() {
Expand Down
21 changes: 18 additions & 3 deletions scripts/gallery-audit/test/audit.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { auditCatalog, checkUrl, discoverArticles, discoverContent, discoverFromFeed, findDuplicates, validateCatalog } from '../core.mjs';
import { buildClassificationPrompt, buildCopilotArguments, runCopilotClassification } from '../copilot.mjs';
import { buildClassificationPrompt, buildCopilotArguments, extractJsonObject, runCopilotClassification } from '../copilot.mjs';
import { urlFingerprint } from '../normalize.mjs';
import { planCatalogPromotion, promotionMarkdown, sortCatalogForPublishing, strongRetirementEvidence, validatePromotionResult } from '../promotion.mjs';

Expand Down Expand Up @@ -560,6 +560,22 @@ review.",
assert.equal(result.classification.newContent[0].evidence, 'Needs\nreview.');
});

test('extracts a JSON object from Copilot response wrappers', () => {
const response = '{"newContent":[],"existingContent":[]}';
assert.equal(extractJsonObject(`'${response}'`), response);
assert.equal(extractJsonObject(`Result:\n${response}\nDone.`), response);
});

test('accepts a single-quoted Copilot response wrapper', () => {
const response = '{"newContent":[],"existingContent":[]}';
const result = runCopilotClassification({
prompt: 'prompt', candidatePath: 'candidates.json', auditPath: 'audit.json', catalogPath: 'catalog.json',
candidates: [], catalog: [], execute: () => ({ status: 0, stdout: `'${response}'` }),
});
assert.equal(result.status, 'complete');
assert.equal(result.attempts, 1);
});

test('embeds JSON inputs as untrusted prompt data without native attachments', () => {
const directory = mkdtempSync(path.join(tmpdir(), 'gallery-copilot-'));
const documents = [
Expand Down Expand Up @@ -594,8 +610,7 @@ test('embeds JSON inputs as untrusted prompt data without native attachments', (
assert.match(prompt, /Keep this/);
const retirementSection = prompt.match(/BEGIN RETIREMENT CANDIDATES ---([\s\S]*?)--- END RETIREMENT CANDIDATES/)?.[1] ?? '';
assert.doesNotMatch(retirementSection, /Keep this/);
assert.equal(argumentsList[0], '-p');
assert.equal(argumentsList[1], prompt);
assert.doesNotMatch(argumentsList.join(' '), /(^| )--prompt( |$)|(^| )-p( |$)/);
assert.ok(argumentsList.includes('--no-color'));
assert.deepEqual(argumentsList.slice(argumentsList.indexOf('--dynamic-retrieval'), argumentsList.indexOf('--dynamic-retrieval') + 2), ['--dynamic-retrieval', 'skills=on']);
assert.equal(argumentsList.some((argument) => argument.startsWith('--attachment')), false);
Expand Down
1 change: 1 addition & 0 deletions scripts/gallery-audit/test/configuration.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ test('keeps audit read-only and publishes only an issue through trusted workflow
assert.match(audit, /r\.additions\?\.length[\s\S]*r\.updates\?\.length[\s\S]*r\.retirements\?\.length/);
assert.match(audit, /steps\.actionable\.outputs\.available == 'true'/);
assert.doesNotMatch(audit, /contents: write|issues: write|git push|gh issue create/);
assert.match(read('scripts/gallery-audit/index.mjs'), /Copilot classification incomplete after/);
Comment thread
jaydestro marked this conversation as resolved.

const publisher = read('.github/workflows/publish-gallery-proposal.yml');
assert.match(publisher, /workflow_run:[\s\S]*workflows: \[Audit gallery content\][\s\S]*types: \[completed\]/);
Expand Down
Loading