From a7a5d40ff3a1c9b8f17aa6fdc018c4fc6e52e28a Mon Sep 17 00:00:00 2001 From: Jay Gordon Date: Tue, 29 Sep 2026 14:50:45 -0400 Subject: [PATCH 1/2] fix: handle wrapped gallery classification output Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/gallery-audit/copilot.mjs | 10 +++++++++- scripts/gallery-audit/index.mjs | 3 ++- scripts/gallery-audit/test/audit.test.mjs | 18 +++++++++++++++++- .../gallery-audit/test/configuration.test.mjs | 1 + 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/scripts/gallery-audit/copilot.mjs b/scripts/gallery-audit/copilot.mjs index 8296be4..8036527 100644 --- a/scripts/gallery-audit/copilot.mjs +++ b/scripts/gallery-audit/copilot.mjs @@ -62,8 +62,16 @@ export function stripJsonFence(value) { return match ? match[1].trim() : trimmed; } +export function extractJsonObject(value) { + const stripped = stripJsonFence(stripVTControlCharacters(value)); + const start = stripped.indexOf('{'); + const end = stripped.lastIndexOf('}'); + if (start === -1 || end < start) return stripped; + return stripped.slice(start, end + 1); +} + function normalizeCopilotJson(value) { - return jsonrepair(stripJsonFence(stripVTControlCharacters(value))); + return jsonrepair(extractJsonObject(value)); } function exactKeys(value, expected) { diff --git a/scripts/gallery-audit/index.mjs b/scripts/gallery-audit/index.mjs index 8cda9bc..7a9568e 100644 --- a/scripts/gallery-audit/index.mjs +++ b/scripts/gallery-audit/index.mjs @@ -90,7 +90,8 @@ async function classifyOnly({ promote = false } = {}) { metadata.copilot = { status: 'incomplete', attempts: result.attempts, error: result.error }; } await writeReports(auditReport, candidateReport, metadata); - if (promote && result.status === 'complete') await applyCatalogPromotion({ root }); + if (result.status !== 'complete') throw new Error(`Copilot classification incomplete after ${result.attempts} attempts: ${result.error}`); + if (promote) await applyCatalogPromotion({ root }); } async function run() { diff --git a/scripts/gallery-audit/test/audit.test.mjs b/scripts/gallery-audit/test/audit.test.mjs index 935b6f3..5aff78f 100644 --- a/scripts/gallery-audit/test/audit.test.mjs +++ b/scripts/gallery-audit/test/audit.test.mjs @@ -6,7 +6,7 @@ import { mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { auditCatalog, checkUrl, discoverArticles, discoverContent, discoverFromFeed, findDuplicates, validateCatalog } from '../core.mjs'; -import { buildClassificationPrompt, buildCopilotArguments, runCopilotClassification } from '../copilot.mjs'; +import { buildClassificationPrompt, buildCopilotArguments, extractJsonObject, runCopilotClassification } from '../copilot.mjs'; import { urlFingerprint } from '../normalize.mjs'; import { planCatalogPromotion, promotionMarkdown, sortCatalogForPublishing, strongRetirementEvidence, validatePromotionResult } from '../promotion.mjs'; @@ -560,6 +560,22 @@ review.", assert.equal(result.classification.newContent[0].evidence, 'Needs\nreview.'); }); +test('extracts a JSON object from Copilot response wrappers', () => { + const response = '{"newContent":[],"existingContent":[]}'; + assert.equal(extractJsonObject(`'${response}'`), response); + assert.equal(extractJsonObject(`Result:\n${response}\nDone.`), response); +}); + +test('accepts a single-quoted Copilot response wrapper', () => { + const response = '{"newContent":[],"existingContent":[]}'; + const result = runCopilotClassification({ + prompt: 'prompt', candidatePath: 'candidates.json', auditPath: 'audit.json', catalogPath: 'catalog.json', + candidates: [], catalog: [], execute: () => ({ status: 0, stdout: `'${response}'` }), + }); + assert.equal(result.status, 'complete'); + assert.equal(result.attempts, 1); +}); + test('embeds JSON inputs as untrusted prompt data without native attachments', () => { const directory = mkdtempSync(path.join(tmpdir(), 'gallery-copilot-')); const documents = [ diff --git a/scripts/gallery-audit/test/configuration.test.mjs b/scripts/gallery-audit/test/configuration.test.mjs index 3be0eb0..1418a02 100644 --- a/scripts/gallery-audit/test/configuration.test.mjs +++ b/scripts/gallery-audit/test/configuration.test.mjs @@ -56,6 +56,7 @@ test('keeps audit read-only and publishes only an issue through trusted workflow assert.match(audit, /r\.additions\?\.length[\s\S]*r\.updates\?\.length[\s\S]*r\.retirements\?\.length/); assert.match(audit, /steps\.actionable\.outputs\.available == 'true'/); assert.doesNotMatch(audit, /contents: write|issues: write|git push|gh issue create/); + assert.match(read('scripts/gallery-audit/index.mjs'), /Copilot classification incomplete after/); const publisher = read('.github/workflows/publish-gallery-proposal.yml'); assert.match(publisher, /workflow_run:[\s\S]*workflows: \[Audit gallery content\][\s\S]*types: \[completed\]/); From 369c1fd8f6c96b505ba67f3c6fb58f12493876ed Mon Sep 17 00:00:00 2001 From: Jay Gordon Date: Tue, 29 Sep 2026 15:05:12 -0400 Subject: [PATCH 2/2] test: replay gallery classification via stdin Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/gallery-audit/copilot.mjs | 5 ++++- scripts/gallery-audit/test/audit.test.mjs | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/gallery-audit/copilot.mjs b/scripts/gallery-audit/copilot.mjs index 8036527..8eb8589 100644 --- a/scripts/gallery-audit/copilot.mjs +++ b/scripts/gallery-audit/copilot.mjs @@ -137,8 +137,8 @@ export function buildClassificationPrompt({ prompt, candidatePath, auditPath, ca } export function buildCopilotArguments({ prompt, candidatePath, auditPath, catalogPath, existingEntries }) { + buildClassificationPrompt({ prompt, candidatePath, auditPath, catalogPath, existingEntries }); return [ - '-p', buildClassificationPrompt({ prompt, candidatePath, auditPath, catalogPath, existingEntries }), '--agent=gallery-curator', '--silent', '--stream=off', @@ -155,6 +155,7 @@ function invokeCopilot(options) { return spawnSync('copilot', buildCopilotArguments(options), { encoding: 'utf8', env: process.env, + input: buildClassificationPrompt(options), maxBuffer: 2 * 1024 * 1024, windowsHide: true, }); @@ -168,6 +169,8 @@ export function runCopilotClassification(options) { for (let attempt = 1; attempt <= 2; attempt += 1) { try { const result = execute(options); + if (result.error) throw result.error; + if (result.signal) throw new Error(`Copilot terminated by signal ${result.signal}`); if (result.status !== 0) throw new Error(`Copilot exited with status ${result.status}: ${(result.stderr ?? '').trim()}`); const parsed = JSON.parse(normalizeCopilotJson(result.stdout ?? '')); return { status: 'complete', attempts: attempt, classification: validateClassification(parsed, options.candidates, existingEntries) }; diff --git a/scripts/gallery-audit/test/audit.test.mjs b/scripts/gallery-audit/test/audit.test.mjs index 5aff78f..439f87a 100644 --- a/scripts/gallery-audit/test/audit.test.mjs +++ b/scripts/gallery-audit/test/audit.test.mjs @@ -610,8 +610,7 @@ test('embeds JSON inputs as untrusted prompt data without native attachments', ( assert.match(prompt, /Keep this/); const retirementSection = prompt.match(/BEGIN RETIREMENT CANDIDATES ---([\s\S]*?)--- END RETIREMENT CANDIDATES/)?.[1] ?? ''; assert.doesNotMatch(retirementSection, /Keep this/); - assert.equal(argumentsList[0], '-p'); - assert.equal(argumentsList[1], prompt); + assert.doesNotMatch(argumentsList.join(' '), /(^| )--prompt( |$)|(^| )-p( |$)/); assert.ok(argumentsList.includes('--no-color')); assert.deepEqual(argumentsList.slice(argumentsList.indexOf('--dynamic-retrieval'), argumentsList.indexOf('--dynamic-retrieval') + 2), ['--dynamic-retrieval', 'skills=on']); assert.equal(argumentsList.some((argument) => argument.startsWith('--attachment')), false);