diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 70d77fa..0d9deb2 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -67,17 +67,32 @@ class Documentarian { const message = error instanceof Error ? error.message : String(error); tag('warning').log(`Interactive documentation failed: ${message}.`); if (meaningfulInteractions.length > 0) { - tag('info').log(`Preserving ${meaningfulInteractions.length} observed interaction(s) without AI summary.`); - return this.normalizeDocumentation( - { - summary: `Observed ${meaningfulInteractions.length} interaction(s); AI-generated summary was unavailable.`, - can: [], - might: [], - interactions: meaningfulInteractions, - }, - state, - research - ); + tag('info').log(`Retrying static documentation while preserving ${meaningfulInteractions.length} observed interaction(s).`); + return this.documentStatic(state, research) + .then((documentation) => + this.normalizeDocumentation( + { + ...documentation, + interactions: meaningfulInteractions, + }, + state, + research + ) + ) + .catch((fallbackError) => { + const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError); + tag('warning').log(`Static documentation fallback failed: ${fallbackMessage}. Preserving observed interactions without AI summary.`); + return this.normalizeDocumentation( + { + summary: `Observed ${meaningfulInteractions.length} interaction(s); AI-generated summary was unavailable.`, + can: [], + might: [], + interactions: meaningfulInteractions, + }, + state, + research + ); + }); } return this.documentStatic(state, research); } @@ -113,13 +128,13 @@ class Documentarian { }, ]; - const response = await this.provider.generateObject(messages, pageDocumentationSchema, undefined, { + const response = await this.provider.generateObject(messages, generatedPageDocumentationSchema, undefined, { agentName: 'documentarian', }); return this.normalizeDocumentation( { - ...(response.object as PageDocumentation), + ...(response.object as GeneratedPageDocumentation), interactions, }, state, @@ -139,11 +154,11 @@ class Documentarian { }, ]; - const response = await this.provider.generateObject(messages, pageDocumentationSchema, undefined, { + const response = await this.provider.generateObject(messages, generatedPageDocumentationSchema, undefined, { agentName: 'documentarian', }); - return this.normalizeDocumentation(response.object as PageDocumentation, state, research); + return this.normalizeDocumentation(response.object as GeneratedPageDocumentation, state, research); } private getSystemPrompt(): string { @@ -259,7 +274,7 @@ class Documentarian { return message.includes('Failed to generate JSON') || message.includes('Failed to validate JSON') || message.includes('failed_generation') || message.includes('No object generated') || message.includes('response did not match schema'); } - private normalizeDocumentation(documentation: PageDocumentation, _state: WebPageState, _research: string): PageDocumentation { + private normalizeDocumentation(documentation: GeneratedPageDocumentation & Partial>, _state: WebPageState, _research: string): PageDocumentation { const normalized = { ...documentation }; if (!normalized.interactions) { normalized.interactions = undefined; @@ -267,10 +282,10 @@ class Documentarian { const qualityNotes = this.evaluateDocumentationQuality(normalized); - return { + return pageDocumentationSchema.parse({ ...normalized, qualityNotes, - }; + }); } private evaluateDocumentationQuality(documentation: PageDocumentation): string[] { @@ -346,39 +361,54 @@ const stateTransitionSchema = z.object({ action: z.string(), before: z.string(), after: z.string(), - targetUrl: z.string().nullable(), - discoveredUrls: z.array(z.string()).nullable(), - newCapabilities: z.array(z.string()).nullable(), + targetUrl: z.string().optional(), + discoveredUrls: z.array(z.string()).optional(), + newCapabilities: z.array(z.string()).optional(), element: z .object({ role: z.string(), name: z.string(), section: z.string(), - container: z.string().nullable(), - locator: z.string().nullable(), + container: z.string().optional(), + locator: z.string().optional(), }) - .nullable(), + .optional(), changes: z .object({ urlChanged: z.boolean(), newElements: z.number(), removedElements: z.number(), }) - .nullable(), + .optional(), + targetState: z + .object({ + kind: z.enum(['page', 'dialog', 'modal', 'section']), + label: z.string(), + url: z.string(), + }) + .optional(), + screenshot: z + .object({ + title: z.string(), + relativePath: z.string(), + }) + .optional(), }); -const pageDocumentationSchema = z.object({ +const generatedPageDocumentationSchema = z.object({ summary: z.string(), can: z.array(capabilitySchema), might: z.array(capabilitySchema), - interactions: z.array(stateTransitionSchema).nullable(), +}); + +const pageDocumentationSchema = generatedPageDocumentationSchema.extend({ + interactions: z.array(stateTransitionSchema).optional(), + qualityNotes: z.array(z.string()).optional(), }); type StateTransition = DocStateTransition; -type PageDocumentation = Omit, 'interactions'> & { - interactions?: StateTransition[]; - qualityNotes?: string[]; -}; +type GeneratedPageDocumentation = z.infer; +type PageDocumentation = z.infer; export { Documentarian }; export type { PageDocumentation, StateTransition }; diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index 39e8359..e481533 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -736,22 +736,20 @@ describe('doc-collector interactive candidate selection', () => { }); describe('documentarian fallback', () => { - it('uses strict-compatible schema for interaction element metadata', async () => { + it('asks the model only for generated documentation fields', async () => { const provider = { - async generateObject(_messages: Array<{ role: string; content: string }>, schema: any) { + async generateObject(messages: Array<{ role: string; content: string }>, schema: any) { const jsonSchema = z.toJSONSchema(schema) as any; - const interaction = jsonSchema.properties.interactions.anyOf[0].items; - const element = interaction.properties.element.anyOf[0]; - expect(interaction.required).toContain('element'); - expect(element.required).toEqual(['role', 'name', 'section', 'container', 'locator']); + expect(jsonSchema.required).toEqual(['summary', 'can', 'might']); + expect(jsonSchema.properties.interactions).toBeUndefined(); + expect(messages[1].content).not.toContain('interactions:'); return { object: { summary: 'Static page', can: [], might: [], - interactions: null, }, }; }, @@ -904,7 +902,6 @@ describe('documentarian interactive mode', () => { summary: 'Suites page', can: [{ action: 'user can import tests', scope: 'page-level', evidence: 'dialog observed' }], might: [], - interactions: null, }, }; }, @@ -959,7 +956,6 @@ describe('documentarian interactive mode', () => { summary: 'Films page', can: [{ action: 'user can browse films', scope: 'list of items', evidence: 'film tiles visible' }], might: [], - interactions: null, }, }; }, @@ -1155,7 +1151,7 @@ describe('documentarian interactive mode', () => { expect(result.summary).toBe('Static fallback'); }); - it('preserves observed interactions when interactive documentation fails JSON validation', async () => { + it('combines static documentation with observed interactions when interactive documentation fails JSON validation', async () => { const provider = { async generateObject(messages: Array<{ role: string; content: string }>) { const prompt = messages[1].content; @@ -1229,7 +1225,14 @@ describe('documentarian interactive mode', () => { ` ); - expect(result.summary).toBe('Observed 1 interaction(s); AI-generated summary was unavailable.'); + expect(result.summary).toBe('Static fallback'); + expect(result.can).toEqual([ + { + action: 'user can view content', + scope: 'page-level', + evidence: 'fallback after invalid interactive JSON', + }, + ]); expect(result.interactions).toHaveLength(1); });