From 2e763893fca6df218322d9ae09560b26706f4d90 Mon Sep 17 00:00:00 2001 From: ZeroY Date: Sat, 19 Sep 2026 10:45:18 +0800 Subject: [PATCH 1/2] feat(providers): opt in to provider-hosted web search per model Add a two-state per-model binding that attaches the vendor's hosted web search tool on the anthropic-messages and openai-responses wires. The pi-ai patch attaches the tool, extracts search blocks and citations from the stream, and replays them verbatim on later turns; a pi-agent-core patch forwards the new progress event so each search round renders live as its own transcript activity row. Search state normalizes to UiMessage.hostedSearch (one round per provider call), persists as an additive transcript block without a SQL migration, and degrades honestly on endpoints that accept but never execute the tool. Recorded in ADR 0291. --- .../settings/ModelSelectionPanes.tsx | 43 ++ .../settings/ProviderSetupDialog.tsx | 1 + .../settings/VendorAccountDialog.tsx | 1 + .../src/components/settings/provider-copy.ts | 1 + .../chat/transcript/ActivityGroup.tsx | 12 + .../chat/transcript/HostedSearchRow.tsx | 146 ++++++ apps/desktop/src/lib/assistant-turns.ts | 12 + apps/desktop/src/styles/messages.css | 52 ++ .../transcript-disclosure-reading.test.mjs | 8 +- crates/host-core/src/plugin_sessions.rs | 1 + crates/host-core/src/plugins/providers.rs | 1 + crates/host-core/src/providers/catalog.rs | 3 + crates/host-core/src/providers/model.rs | 5 + crates/host-core/src/providers/tests.rs | 3 + crates/host-core/src/sessions.rs | 122 +++++ ...er-hosted-web-search-adapter-capability.md | 105 ++++ docs/adr/README.md | 1 + docs/spec/03-runtime/04-data-storage.md | 8 +- .../03-runtime/11-provider-model-system.md | 9 + docs/zh-CN/spec/03-runtime/04-data-storage.md | 8 +- .../03-runtime/11-provider-model-system.md | 7 + .../src/hosted-search-contract.test.ts | 466 ++++++++++++++++++ .../agent-runtime/src/model-capabilities.ts | 2 + packages/agent-runtime/src/runtime.test.ts | 92 ++++ packages/agent-runtime/src/runtime.ts | 46 ++ packages/agent-runtime/src/thinking-level.ts | 7 + packages/i18n/src/locales/de/index.ts | 12 + packages/i18n/src/locales/en/index.ts | 12 + packages/i18n/src/locales/es/index.ts | 12 + packages/i18n/src/locales/fr/index.ts | 12 + packages/i18n/src/locales/ko/index.ts | 12 + packages/i18n/src/locales/tr/index.ts | 12 + packages/i18n/src/locales/zh-CN/index.ts | 12 + packages/i18n/src/locales/zh-TW/index.ts | 12 + packages/shared/src/index.ts | 1 + packages/shared/src/message-stream.ts | 6 + packages/shared/src/native-web-search.test.ts | 465 +++++++++++++++++ packages/shared/src/native-web-search.ts | 332 +++++++++++++ packages/shared/src/types/messages.ts | 43 ++ packages/shared/src/types/models.ts | 7 + ...earendil-works__pi-agent-core@0.85.1.patch | 17 + patches/@earendil-works__pi-ai@0.85.1.patch | 288 ++++++++++- pnpm-lock.yaml | 19 +- pnpm-workspace.yaml | 1 + 44 files changed, 2420 insertions(+), 17 deletions(-) create mode 100644 apps/desktop/src/features/chat/transcript/HostedSearchRow.tsx create mode 100644 docs/adr/0294-provider-hosted-web-search-adapter-capability.md create mode 100644 packages/agent-runtime/src/hosted-search-contract.test.ts create mode 100644 packages/shared/src/native-web-search.test.ts create mode 100644 packages/shared/src/native-web-search.ts create mode 100644 patches/@earendil-works__pi-agent-core@0.85.1.patch diff --git a/apps/desktop/src/components/settings/ModelSelectionPanes.tsx b/apps/desktop/src/components/settings/ModelSelectionPanes.tsx index 5a9effaeef..78e88a77df 100644 --- a/apps/desktop/src/components/settings/ModelSelectionPanes.tsx +++ b/apps/desktop/src/components/settings/ModelSelectionPanes.tsx @@ -182,6 +182,12 @@ export type ModelSelectionPanesProps = { busy?: boolean; /** Probe the service's model list now, skipping the edit debounce. */ onReload?: () => void; + /** + * Effective API style of the provider being configured. Gates the native + * web search opt-in: only wires that can carry a provider-hosted search + * tool offer the checkbox at all. + */ + apiStyle?: string; }; /** @@ -195,6 +201,7 @@ export function ModelSelectionPanes({ listTitle, busy = false, onReload, + apiStyle, }: ModelSelectionPanesProps) { const { t } = useTranslation(); const { rows, models, publishedLevelsById, setModels } = selection; @@ -245,6 +252,12 @@ export function ModelSelectionPanes({ if (models.length === 0) setChosenQuery(""); }, [models.length]); + // The hosted web search tool only exists on two wire APIs; on any other + // style the opt-in cannot work, so the checkbox stays present but disabled + // with an explanatory hint instead of silently doing nothing. + const nativeWebSearchWireCapable = + apiStyle === "responses" || apiStyle === "anthropic_messages"; + /** * The chosen list narrows with the discovered list's rule plus the binding's * alias: a case-insensitive substring match over the id, the alias, and the @@ -805,6 +818,36 @@ export function ModelSelectionPanes({ + + + + + + diff --git a/apps/desktop/src/components/settings/ProviderSetupDialog.tsx b/apps/desktop/src/components/settings/ProviderSetupDialog.tsx index f36723c626..e4fc045ffc 100644 --- a/apps/desktop/src/components/settings/ProviderSetupDialog.tsx +++ b/apps/desktop/src/components/settings/ProviderSetupDialog.tsx @@ -527,6 +527,7 @@ export function ProviderSetupDialog({ listTitle={t("settings.serviceModels")} busy={saving} onReload={discovery.reload} + apiStyle={resolvedApiStyle} /> diff --git a/apps/desktop/src/components/settings/VendorAccountDialog.tsx b/apps/desktop/src/components/settings/VendorAccountDialog.tsx index 420353155a..bd29d6e636 100644 --- a/apps/desktop/src/components/settings/VendorAccountDialog.tsx +++ b/apps/desktop/src/components/settings/VendorAccountDialog.tsx @@ -138,6 +138,7 @@ export function VendorAccountDialog({ listTitle={t("settings.accountModels")} busy={saving} onReload={discovery.reload} + apiStyle={provider.apiStyle ?? ""} /> diff --git a/apps/desktop/src/components/settings/provider-copy.ts b/apps/desktop/src/components/settings/provider-copy.ts index 81874d83df..653980622c 100644 --- a/apps/desktop/src/components/settings/provider-copy.ts +++ b/apps/desktop/src/components/settings/provider-copy.ts @@ -40,6 +40,7 @@ export function copyProviderConfiguration(provider: ProviderPublic, name: string ...(model.supportsImages !== undefined ? { supportsImages: model.supportsImages } : {}), ...(model.supportsDocuments !== undefined ? { supportsDocuments: model.supportsDocuments } : {}), ...(model.availableForSubagents !== undefined ? { availableForSubagents: model.availableForSubagents } : {}), + ...(model.nativeWebSearch !== undefined ? { nativeWebSearch: model.nativeWebSearch } : {}), })), }; } diff --git a/apps/desktop/src/features/chat/transcript/ActivityGroup.tsx b/apps/desktop/src/features/chat/transcript/ActivityGroup.tsx index 201ca15bd9..47778840d4 100644 --- a/apps/desktop/src/features/chat/transcript/ActivityGroup.tsx +++ b/apps/desktop/src/features/chat/transcript/ActivityGroup.tsx @@ -55,12 +55,14 @@ import { ToolRow } from "./ToolRow"; import { TranscriptSearchContext } from "../../../lib/transcript-search-context"; import { useAppStore } from "../../../stores/app-store"; import { resolveThinkingDisplayMode } from "../../../lib/turn-process"; +import { HostedSearchRow } from "./HostedSearchRow"; type Translate = (key: string, options?: Record) => string; type ActivityItem = AssistantActivityItem; export function activityItemDetail(item: ActivityItem): string { + if (item.kind === "hostedSearch") return item.round.query ?? ""; if (item.kind === "thinking") { // Latest thought line, so a collapsed header reads like a live ticker. const lines = thinkingText(item.message) @@ -169,6 +171,9 @@ export function activityItemsEqual( if (previous.kind === "tool" && next.kind === "tool") { return subagentRunsEqual(previous.delegate, next.delegate); } + if (previous.kind === "hostedSearch" && next.kind === "hostedSearch") { + return previous.round === next.round; + } return true; } @@ -349,6 +354,13 @@ export const ActivityGroup = memo(function ActivityGroup({ /> + ) : item.kind === "hostedSearch" ? ( + ) : ( void; +}) { + const { t } = useTranslation(); + const detailsId = useId(); + const disclosure = useAutomaticDisclosure(false); + const { open, toggle: toggleDisclosure, collapse: collapseDisclosure } = disclosure; + const titleRef = disclosure.titleRef; + const toggleRow = useCallback(() => { + onUserInteraction?.(); + toggleDisclosure(); + }, [onUserInteraction, toggleDisclosure]); + const collapseRow = useCallback(() => { + onUserInteraction?.(); + collapseDisclosure(); + }, [collapseDisclosure, onUserInteraction]); + + const searching = streaming && round.status === "searching"; + const failed = round.status === "failed"; + const sources = round.sources ?? []; + // The opened page leads the body list; dedupe against extracted sources. + const links = [ + ...(round.url ? [{ url: round.url }] : []), + ...sources.filter((source) => source.url !== round.url), + ]; + const shown = links.slice(0, HOSTED_SEARCH_PREVIEW_COUNT); + const hidden = links.length - shown.length; + const expandable = Boolean(round.query) || links.length > 0; + const summary = + round.query ?? + (round.url + ? sourceHost(round.url) + : sources.length > 0 + ? t("chat.webSearchSources", { count: sources.length }) + : ""); + const name = failed + ? t("chat.webSearchFailed") + : searching + ? t("chat.webSearching") + : round.kind === "openPage" + ? t("chat.webOpenPage") + : round.kind === "findInPage" + ? t("chat.webFindInPage") + : t("chat.webSearch"); + + return ( +
+ + {open && expandable ? ( +
+ + {round.query ? ( +
{round.query}
+ ) : null} + {shown.length > 0 ? ( + + ) : null} + {hidden > 0 ? ( + + {t("chat.webSearchMore", { count: hidden })} + + ) : null} +
+ ) : null} +
+ ); +}, (previous, next) => + previous.round === next.round && + previous.streaming === next.streaming && + previous.onUserInteraction === next.onUserInteraction, +); diff --git a/apps/desktop/src/lib/assistant-turns.ts b/apps/desktop/src/lib/assistant-turns.ts index 86fbc4c439..60cd553740 100644 --- a/apps/desktop/src/lib/assistant-turns.ts +++ b/apps/desktop/src/lib/assistant-turns.ts @@ -1,8 +1,10 @@ import type { ContextCompactionMark, + HostedSearchRound, MessageUsage, UiMessage, } from "@pi-desktop/shared"; +import { hostedSearchRounds } from "@pi-desktop/shared"; import { isDelegationStartTool } from "./tool-display"; export type AssistantActivityItem = @@ -12,6 +14,12 @@ export type AssistantActivityItem = message: UiMessage; /** Present on a `Task` call: what the delegate it spawned did. */ delegate?: SubagentRun; + } + | { + kind: "hostedSearch"; + message: UiMessage; + /** One provider search round of the message; each round is a row. */ + round: HostedSearchRound; }; /** One row a delegate produced, in the order the delegate produced it. */ @@ -64,6 +72,7 @@ function isVisibleMessage(message: UiMessage): boolean { message.role === "assistant" && !(message.content || "").trim() && !messageThinking(message) && + !message.hostedSearch && !message.error ); } @@ -241,6 +250,9 @@ export function buildTranscriptEntries( const current = ensureTurn(message); const thinking = messageThinking(message); if (thinking) pushActivity({ kind: "thinking", message }); + for (const round of hostedSearchRounds(message.hostedSearch)) { + pushActivity({ kind: "hostedSearch", message, round }); + } if ((message.content || "").trim() || !thinking || message.error) { current.parts.push({ kind: "message", message }); if (!current.anchorId && (message.content || "").trim()) { diff --git a/apps/desktop/src/styles/messages.css b/apps/desktop/src/styles/messages.css index 38cb624e19..5052c52a7d 100644 --- a/apps/desktop/src/styles/messages.css +++ b/apps/desktop/src/styles/messages.css @@ -2981,3 +2981,55 @@ display: inline-flex; color: var(--ds-error); } + +/* Provider-hosted web search round. The row reuses the tool-row idiom; only + * the body payload needs its own rules. */ +.hosted-search-query { + margin: 2px 0 6px; + color: var(--ds-text-secondary); + font-size: var(--text-sm); + white-space: pre-wrap; + word-break: break-word; +} + +.hosted-search-sources { + margin: 0; + padding: 0; + list-style: none; + display: flex; + flex-direction: column; + gap: 4px; +} + +.hosted-search-sources a { + display: inline-flex; + align-items: baseline; + gap: 6px; + max-width: 100%; + min-width: 0; + color: var(--ds-text-primary); + font-size: var(--text-sm); + text-decoration: none; +} + +.hosted-search-sources a:hover { + text-decoration: underline; +} + +.hosted-search-source-title { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.hosted-search-source-host { + color: var(--ds-text-tertiary); + flex: none; +} + +.hosted-search-more { + display: inline-block; + margin-top: 4px; + color: var(--ds-text-tertiary); + font-size: var(--text-sm); +} diff --git a/apps/desktop/test/transcript-disclosure-reading.test.mjs b/apps/desktop/test/transcript-disclosure-reading.test.mjs index 15ac5f40a1..fdeeec38ac 100644 --- a/apps/desktop/test/transcript-disclosure-reading.test.mjs +++ b/apps/desktop/test/transcript-disclosure-reading.test.mjs @@ -54,8 +54,8 @@ test("every manual title hands over the element the reader clicked", () => { ); assert.equal( toolHeaders?.length, - 2, - "the thinking row and the tool row must both anchor their own header", + 3, + "the thinking row, the tool row and the hosted search row must each anchor their own header", ); assert.match(transcript, /onCollapse=\{collapseDisclosure\}/); const collapseWrappers = transcript.match( @@ -63,8 +63,8 @@ test("every manual title hands over the element the reader clicked", () => { ); assert.equal( collapseWrappers?.length, - 2, - "the thinking row and the tool row must both anchor when their rail collapses them", + 3, + "the thinking row, the tool row and the hosted search row must each anchor when their rail collapses them", ); }); diff --git a/crates/host-core/src/plugin_sessions.rs b/crates/host-core/src/plugin_sessions.rs index 38862f0eec..fd09e435a4 100644 --- a/crates/host-core/src/plugin_sessions.rs +++ b/crates/host-core/src/plugin_sessions.rs @@ -255,6 +255,7 @@ fn parse_message( is_error: None, parent_tool_call_id: None, agent_name: None, + hosted_search: None, session_message: None, }) } diff --git a/crates/host-core/src/plugins/providers.rs b/crates/host-core/src/plugins/providers.rs index 5c053fd766..d55a1954cc 100644 --- a/crates/host-core/src/plugins/providers.rs +++ b/crates/host-core/src/plugins/providers.rs @@ -145,6 +145,7 @@ pub(crate) fn declared_providers(manifest: &PluginManifest) -> Vec>() diff --git a/crates/host-core/src/providers/catalog.rs b/crates/host-core/src/providers/catalog.rs index 078a09a2dc..834f0042e9 100644 --- a/crates/host-core/src/providers/catalog.rs +++ b/crates/host-core/src/providers/catalog.rs @@ -62,6 +62,7 @@ pub(crate) fn normalize_model_bindings(bindings: &[ModelBinding]) -> Vec) -> Vec { supports_images: None, supports_documents: None, available_for_subagents: None, + native_web_search: None, }] }) .unwrap_or_default() @@ -235,6 +237,7 @@ mod tests { supports_images: None, supports_documents: None, available_for_subagents: None, + native_web_search: None, } } diff --git a/crates/host-core/src/providers/model.rs b/crates/host-core/src/providers/model.rs index b218c47adb..ad037bda3d 100644 --- a/crates/host-core/src/providers/model.rs +++ b/crates/host-core/src/providers/model.rs @@ -149,6 +149,11 @@ pub struct ModelBinding { /// None/false keeps the opt-in disabled for existing provider records. #[serde(default, skip_serializing_if = "Option::is_none")] pub available_for_subagents: Option, + /// Opt-in for attaching the provider-hosted web search tool to requests + /// for this model. None/false keeps the tool off; there is no catalog + /// default because models.dev does not publish hosted-tool capability. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_web_search: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] diff --git a/crates/host-core/src/providers/tests.rs b/crates/host-core/src/providers/tests.rs index 11fd1d5e70..4bd6c568d1 100644 --- a/crates/host-core/src/providers/tests.rs +++ b/crates/host-core/src/providers/tests.rs @@ -170,6 +170,7 @@ fn model_bindings_roundtrip_and_legacy_model_migrates_on_read() { supports_images: Some(true), supports_documents: None, available_for_subagents: Some(true), + native_web_search: None, }, ModelBinding { id: "plain-model".into(), @@ -182,6 +183,7 @@ fn model_bindings_roundtrip_and_legacy_model_migrates_on_read() { supports_images: None, supports_documents: Some(false), available_for_subagents: None, + native_web_search: None, }, ]), default_model_id: None, @@ -283,6 +285,7 @@ fn binding_with_alias(id: &str, alias: Option<&str>) -> ModelBinding { supports_images: None, supports_documents: None, available_for_subagents: None, + native_web_search: None, } } diff --git a/crates/host-core/src/sessions.rs b/crates/host-core/src/sessions.rs index 5ab628632f..15f8546a77 100644 --- a/crates/host-core/src/sessions.rs +++ b/crates/host-core/src/sessions.rs @@ -204,6 +204,10 @@ pub struct UiMessage { /// Subagent definition name that produced the row. #[serde(skip_serializing_if = "Option::is_none")] pub agent_name: Option, + /// Provider-hosted web search activity for this assistant turn. Persisted + /// as an additive `hostedSearch` transcript block; no SQL migration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hosted_search: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -337,6 +341,18 @@ pub(crate) fn ui_to_record(message: &UiMessage) -> (MessageRecord, Option UiMessage { }) .collect::>(); let thinking = (!thinking.is_empty()).then(|| thinking.concat()); + let hosted_search = blocks + .iter() + .find(|b| b.get("type").and_then(|t| t.as_str()) == Some("hostedSearch")) + .map(|b| { + let mut fields = b.clone(); + if let Value::Object(map) = &mut fields { + map.remove("type"); + } + fields + }); let is_error = record.is_error.then_some(true); let attachments = blocks .iter() @@ -536,6 +562,7 @@ pub(crate) fn record_to_ui(record: MessageRecord) -> UiMessage { is_error, parent_tool_call_id, agent_name, + hosted_search: hosted_search.clone(), } } else { let content = blocks @@ -575,6 +602,7 @@ pub(crate) fn record_to_ui(record: MessageRecord) -> UiMessage { is_error, parent_tool_call_id, agent_name, + hosted_search, } } } @@ -3433,6 +3461,7 @@ mod tests { is_error: None, parent_tool_call_id: None, agent_name: None, + hosted_search: None, session_message: None, } } @@ -3931,6 +3960,7 @@ mod tests { is_error: None, parent_tool_call_id: None, agent_name: None, + hosted_search: None, session_message: None, }; append_message(&db, &session.id, &tool, None).unwrap(); @@ -4366,6 +4396,7 @@ mod tests { is_error: None, parent_tool_call_id: None, agent_name: None, + hosted_search: None, session_message: None, }; append_message(&db, &session.id, &assistant, None).unwrap(); @@ -4409,6 +4440,97 @@ mod tests { assert_eq!(detail.messages[0].response_output_tokens, Some(34)); } + #[test] + fn assistant_hosted_search_roundtrips_as_canonical_blocks() { + let db = test_db(); + let session = create_session(&db, None, None, None, None, None).unwrap(); + let assistant = UiMessage { + id: "assistant-search-1".into(), + role: "assistant".into(), + content: "answer with sources".into(), + attachments: None, + steering: None, + created_at: "2025-05-01T00:00:01Z".into(), + thinking: None, + status: Some("complete".into()), + model_id: Some("model-1".into()), + provider_id: Some("provider-1".into()), + usage: None, + response_duration_ms: None, + response_output_tokens: None, + error: None, + revision_root_id: None, + revision_count: None, + active_revision: None, + tool_name: None, + tool_call_id: None, + tool_status: None, + tool_args: None, + tool_result: None, + tool_completed_at: None, + tool_duration_ms: None, + is_error: None, + parent_tool_call_id: None, + agent_name: None, + hosted_search: Some(json!({ + "status": "completed", + "rounds": [ + { + "id": "srvtoolu_01", + "status": "completed", + "query": "pi-desktop release notes", + "sources": [ + { "url": "https://example.com/a", "title": "A" } + ] + } + ] + })), + session_message: None, + }; + append_message(&db, &session.id, &assistant, None).unwrap(); + + let records = transcripts::read_transcript(db.data_dir(), &session.id).unwrap(); + assert_eq!(records.len(), 1); + let blocks = &records[0].blocks; + assert_eq!( + blocks[0], + json!({ + "type": "hostedSearch", + "status": "completed", + "rounds": [ + { + "id": "srvtoolu_01", + "status": "completed", + "query": "pi-desktop release notes", + "sources": [ + { "url": "https://example.com/a", "title": "A" } + ] + } + ] + }) + ); + + let detail = get_session(&db, &session.id).unwrap().unwrap(); + assert_eq!( + detail.messages[0].hosted_search, + Some(json!({ + "status": "completed", + "rounds": [ + { + "id": "srvtoolu_01", + "status": "completed", + "query": "pi-desktop release notes", + "sources": [ + { "url": "https://example.com/a", "title": "A" } + ] + } + ] + })) + ); + // The additive block must not disturb text reconstruction. + assert_eq!(detail.messages[0].content, "answer with sources"); + } + #[test] fn assistant_error_roundtrips_in_message_metadata() { let db = test_db(); diff --git a/docs/adr/0294-provider-hosted-web-search-adapter-capability.md b/docs/adr/0294-provider-hosted-web-search-adapter-capability.md new file mode 100644 index 0000000000..3b5b410c33 --- /dev/null +++ b/docs/adr/0294-provider-hosted-web-search-adapter-capability.md @@ -0,0 +1,105 @@ +# ADR 0294: Provider-hosted web search as an adapter capability + +- Status: Proposed +- Date: 2026-09-19 + +## Context + +Some providers execute web search server-side: Anthropic Messages ships a +`web_search_20250305` server tool, and OpenAI Responses (plus compatible +gateways) ships a `web_search` tool. Unlike an MCP search server, the search +runs inside the provider request, is billed by the provider per search, and +returns structured blocks (`server_tool_use` + `web_search_tool_result`, +`web_search_call`, `url_citation` annotations) that the client must keep in +the conversation for multi-turn grounding. + +PR #570 previously added this as a composer toggle with runtime-level SSE +re-parsing; it was reverted (#608) with the owner asking for a plugin-shaped +follow-up. A capability review found the plugin SDK cannot reach the request +payload, the response stream, the composer, the transcript, or message +persistence, so "web search as a plugin" would require four new SDK extension +points — larger than the feature itself. This ADR records the host-built +shape that landed instead. + +Two upstream facts forced the design: + +- pi-ai 0.85.1 drops vendor search blocks in both adapters: the Anthropic + stream loop has no `server_tool_use` / `web_search_tool_result` / + `citations_delta` branches, and the Responses item loop has no + `web_search_call` slot or annotation handling. Extraction must therefore + live inside the adapters, not beside them. +- Anthropic requires `server_tool_use` + `web_search_tool_result` blocks + (including `encrypted_content`) to be replayed verbatim on later turns. + A display-only normalization cannot satisfy that; replay needs the raw + wire blocks. + +## Decision + +1. **Capability is declared data, evaluated in one place.** The per-model + opt-in is `ModelBinding.nativeWebSearch` (settings UI: a checkbox in the + model's advanced sheet, disabled unless the provider's API style is + `responses` or `anthropic_messages`). models.dev publishes no hosted-tool + capability, so there is no catalog default; absent means off. The single + evaluation point is `resolveNativeWebSearch` in + `packages/shared/src/native-web-search.ts`, keyed on the resolved wire + API — never vendor names, base URL hostnames, or model-id substrings. + +2. **Attachment and extraction live in the pi-ai adapters**, delivered by + extending `patches/@earendil-works__pi-ai@0.85.1.patch`: + - `anthropic-messages.js` appends the `web_search_20250305` tool when + `model.webSearch === true`, captures search blocks as `hostedSearch` + content parts (raw wire block kept whole, streamed `input_json_delta` + accumulated so the query is available live), collects `citations_delta`, + and replays both halves of each pair in order on later turns. + - `openai-responses(-shared).js` (and the Azure variant's own + `buildParams`) appends the `web_search` tool, asks for + `web_search_call.action.sources` via the opt-in `include` channel, + creates a `hostedSearch` slot for `web_search_call` items, collects + `url_citation` annotations, and replays the item for the same model. + Both adapters push a `hosted_search_update` stream event per block + transition; `patches/@earendil-works__pi-agent-core@0.85.1.patch` teaches + the agent loop to forward it as `message_update` — without that second + patch the events die in the loop's switch and search activity renders only + when the whole turn finishes. The patches are a stopgap; the same changes + should be proposed upstream and dropped once released. + +3. **The flag travels the existing model-config channel.** + `ModelConfigWithBinding` copies `nativeWebSearch` into + `ModelConfig.webSearch`; `buildProviderModel` spreads catalog fields into + the pi-ai `Model`, so no new IPC, sidecar parameter, or runtime rebuild + hook is needed. + +4. **Display data and replay data are separated.** The runtime normalizes + `hostedSearch` blocks into `UiMessage.hostedSearch` + (`status/rounds[]`, one round per provider search/open-page/find-in-page + call, shape-checked, junk ignored) for the activity rows; the raw wire blocks stay inside + pi-ai's message content for replay. Persistence is an additive + `hostedSearch` transcript block (`ui_to_record` / `record_to_ui`) — no SQL + migration, per the storage playbook. The wire-slimming functions + (`streamingMessageIdentity`, `applyMessageUpdate`) carry the field so + delta frames cannot drop it. + +5. **UI is a transcript activity row per round, not a composer control.** + Each round of a `hostedSearch` activity renders as a `HostedSearchRow` on + the tool-row idiom (icon + name + query summary, chevron disclosure): + searching state while the round is in flight, sources when expanded. + Sources are plain text links — no favicon fetches — so reading a + transcript never leaks source hostnames anywhere (the #579 rule). + Citation badges in message text are out of scope for v1; links stay + ordinary markdown links. + +## Consequences + +- Enabling the tool is per model, always-on while enabled, and takes effect + on the next turn (runtime rebuild follows from modelConfig change). A + session-level toggle and composer affordance can layer on later without + touching the adapter contract. +- Gateways that mangle replay blocks surface as provider errors on the next + turn; the user-visible remedy is unchecking the model's opt-in. No silent + fallback strips the tool — a search that quietly did not happen is worse + than an error. +- `pause_turn` keeps pi-ai's existing mapping to `stop`; a long searching + turn may end early on the official Anthropic wire. This is a known + limitation to revisit with the upstream patch. +- Compaction rewrites history and drops search blocks; later turns lose old + grounding and the model searches again as needed. Documented behavior. diff --git a/docs/adr/README.md b/docs/adr/README.md index e037741a50..40b05967ac 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -320,4 +320,5 @@ Each ADR includes: | 0291 | [Remove the speech settings UI](0291-remove-speech-settings-ui.md) | Accepted (amends ADR 0281) | | 0292 | [SSH bootstrap for remote hosts](0292-ssh-remote-host-bootstrap.md) | Accepted for implementation (D453; ADR 0205 R2b, extends ADR 0286) | | 0293 | [SSH password authentication for the remote-host bootstrap](0293-ssh-password-authentication.md) | Accepted (D454; amends ADR 0292) | +| 0294 | [Provider-hosted web search as an adapter capability](0294-provider-hosted-web-search-adapter-capability.md) | Proposed | | turn-process-and-thinking-display | [Turn process and thinking presentation](turn-process-and-thinking-display.md) | Accepted | diff --git a/docs/spec/03-runtime/04-data-storage.md b/docs/spec/03-runtime/04-data-storage.md index 748e44df6a..bc8cb848f8 100644 --- a/docs/spec/03-runtime/04-data-storage.md +++ b/docs/spec/03-runtime/04-data-storage.md @@ -748,7 +748,13 @@ type Block = toolUsage?: ToolTokenUsage } | { type: "attachment"; kind: "image" | "file"; name: string; ref: string /* attachments/ or absolute path */; - mimeType?: string; size?: number }; + mimeType?: string; size?: number } + | { type: "hostedSearch"; status: "searching" | "completed" | "failed"; + rounds: Array<{ id: string; + status: "searching" | "completed" | "failed"; + kind?: "search" | "openPage" | "findInPage"; + query?: string; url?: string; + sources: Array<{ url: string; title?: string }> }> }; ``` - Tool results are stored **post-truncation** (16-tool-result-limits); full diff --git a/docs/spec/03-runtime/11-provider-model-system.md b/docs/spec/03-runtime/11-provider-model-system.md index 9d3865e298..6d322f7a4a 100644 --- a/docs/spec/03-runtime/11-provider-model-system.md +++ b/docs/spec/03-runtime/11-provider-model-system.md @@ -224,6 +224,15 @@ PI-Desktop must not permanently restrict users to a short fixed model list. catalog" rather than an equal-valued override. Agreeing with models.dev is therefore the reset, and no separate reset control or per-capability explanatory copy is required. +10a. `nativeWebSearch` is a two-state opt-in (absent means off; there is no + catalog baseline because models.dev publishes no hosted-tool capability). + When enabled and the provider's resolved wire API is + `anthropic_messages` or `responses`, the adapter attaches the provider's + hosted web search tool (`web_search_20250305` / `web_search`), extracts + the search activity into `UiMessage.hostedSearch`, and replays the raw + search blocks on later turns (ADR 0294). The checkbox is disabled when + the provider's API style is neither of those two. Gateways that do not + support the tool surface the provider error; the remedy is unchecking. 11. `ModelInfo` is the published record the settings surface compares against, so a stored binding must not shape its capabilities or reasoning fields. Effective limits, reasoning and thinking levels are resolved through the diff --git a/docs/zh-CN/spec/03-runtime/04-data-storage.md b/docs/zh-CN/spec/03-runtime/04-data-storage.md index 7e20d7dcaa..a860a8277c 100644 --- a/docs/zh-CN/spec/03-runtime/04-data-storage.md +++ b/docs/zh-CN/spec/03-runtime/04-data-storage.md @@ -674,7 +674,13 @@ type Block = completedAt?: string; durationMs?: number; toolUsage?: ToolTokenUsage } | { type: "attachment"; kind: "image" | "file"; name: string; - ref: string /* attachments/ or absolute path */ }; + ref: string /* attachments/ or absolute path */ } + | { type: "hostedSearch"; status: "searching" | "completed" | "failed"; + rounds: Array<{ id: string; + status: "searching" | "completed" | "failed"; + kind?: "search" | "openPage" | "findInPage"; + query?: string; url?: string; + sources: Array<{ url: string; title?: string }> }> }; ``` - 工具结果存储**截断后**(16 个工具结果限制);满 diff --git a/docs/zh-CN/spec/03-runtime/11-provider-model-system.md b/docs/zh-CN/spec/03-runtime/11-provider-model-system.md index cda3c60ee5..8b772b6e10 100644 --- a/docs/zh-CN/spec/03-runtime/11-provider-model-system.md +++ b/docs/zh-CN/spec/03-runtime/11-provider-model-system.md @@ -201,6 +201,13 @@ PI-Desktop 不得把用户永久限制在一份简短的固定模型列表上。 10. 设置里的复选框展示的是相对于已发布基线的有效答案;把某一项设回已发布的 值,存下来的是"跟随目录",而不是一个取值相同的覆盖。因此与 models.dev 保持一致本身就是重置,不需要另外的重置控件,也不需要逐项能力的解释文案。 +10a. `nativeWebSearch` 是两态的主动开启(缺省即关闭;没有目录基线,因为 + models.dev 不发布托管工具能力)。启用且提供商解析后的线路 API 是 + `anthropic_messages` 或 `responses` 时,适配器会附加提供商托管的联网 + 搜索工具(`web_search_20250305` / `web_search`),把搜索活动提取为 + `UiMessage.hostedSearch`,并在后续回合回放原始搜索块(ADR 0294)。 + 提供商接口风格不属于这两种时复选框禁用。不支持该工具的网关会把 + 提供商错误暴露出来;处理方式是取消勾选。 11. `ModelInfo` 是设置界面用来对照的已发布记录,因此已存储的 binding 不得 塑造它的能力或推理字段。有效上限、推理与思考级别都通过那个确切的 binding 解析;有效的传输模态数组还会额外套用显式的附件覆盖。 diff --git a/packages/agent-runtime/src/hosted-search-contract.test.ts b/packages/agent-runtime/src/hosted-search-contract.test.ts new file mode 100644 index 0000000000..47509dac2d --- /dev/null +++ b/packages/agent-runtime/src/hosted-search-contract.test.ts @@ -0,0 +1,466 @@ +import { describe, expect, it } from "vitest"; + +import type { AssistantMessage } from "@earendil-works/pi-ai"; + +/** + * Contract tests for the pi-ai hosted web search patch. + * + * The patch (patches/@earendil-works__pi-ai@0.85.1.patch) teaches the + * anthropic-messages and openai-responses adapters to attach the provider + * hosted web search tool when the model record opts in, to extract the search + * blocks and citations from the stream, and to replay the search items on + * later turns. These tests drive the exported stream processor with + * synthetic provider events — including malformed ones — so the extraction + * contract is locked without hitting a real provider. + */ + +type AnyRecord = Record; + +async function processEvents(events: AnyRecord[]): Promise<{ + output: AssistantMessage & { + hostedSearchCitations?: { url: string; title?: string }[]; + }; + stream: AnyRecord[]; +}> { + const { processResponsesStream } = await import("@earendil-works/pi-ai/api/openai-responses-shared"); + const output = { + role: "assistant", + content: [], + api: "openai-responses", + provider: "openai", + model: "gpt-test", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "pending", + timestamp: Date.now(), + }; + const stream: AnyRecord[] = []; + const model = { + id: "gpt-test", + api: "openai-responses", + provider: "openai", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + } as never; + const sink = { + push: (event: AnyRecord) => stream.push(event), + } as unknown as Parameters[2]; + const asyncEvents = (async function* () { + for (const event of events) yield event; + })(); + await processResponsesStream( + asyncEvents as never, + output as unknown as AssistantMessage, + sink, + model, + {}, + ); + return { output: output as unknown as AssistantMessage & { + hostedSearchCitations?: { url: string; title?: string }[]; + }, stream }; +} + +describe("pi-ai hosted web search: responses stream extraction", () => { + it("captures a web_search_call as a hostedSearch content block", async () => { + const { output } = await processEvents([ + { + type: "response.created", + response: { id: "resp_1" }, + }, + { + type: "response.output_item.added", + output_index: 0, + item: { + type: "web_search_call", + id: "ws_1", + status: "in_progress", + action: { type: "search", query: "pi-desktop release notes" }, + }, + }, + { + type: "response.output_item.added", + output_index: 1, + item: { type: "message", id: "msg_1", role: "assistant" }, + }, + { + type: "response.output_text.delta", + output_index: 1, + delta: "Found it.", + }, + { + type: "response.output_text.annotation.added", + output_index: 1, + annotation: { + type: "url_citation", + url: "https://example.com/release", + title: "Release notes", + start_index: 0, + end_index: 5, + }, + }, + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "pi-desktop release notes" }, + }, + }, + { + type: "response.output_item.done", + output_index: 1, + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Found it.", annotations: [] }], + }, + }, + { + type: "response.completed", + response: { id: "resp_1", usage: {} }, + }, + ]); + + const search = output.content.find( + (block) => ((block ?? {}) as unknown as AnyRecord).type === "hostedSearch", + ) as unknown as AnyRecord | undefined; + expect(search).toBeDefined(); + expect(search?.phase).toBe("web_search_call"); + expect(search?.status).toBe("completed"); + expect((((search ?? {}) as AnyRecord).wire as AnyRecord)?.action).toMatchObject({ + type: "search", + query: "pi-desktop release notes", + }); + + const citations = (output as unknown as AnyRecord).hostedSearchCitations as + AnyRecord[] | undefined; + expect(citations).toEqual([ + { url: "https://example.com/release", title: "Release notes" }, + ]); + }); + + it("ignores malformed annotations and unknown web_search_call shapes", async () => { + const { output } = await processEvents([ + { + type: "response.output_text.annotation.added", + output_index: 0, + // Missing url: dropped rather than crashing the turn. + annotation: { type: "url_citation", title: "no url" }, + }, + { + type: "response.output_text.annotation.added", + output_index: 0, + annotation: { type: "something_else", url: "https://example.com" }, + }, + { + type: "response.web_search_call.completed", + // No item payload: nothing to slot, and no error. + output_index: 3, + }, + { + type: "response.output_item.added", + output_index: 0, + item: { type: "message", id: "msg_1", role: "assistant" }, + }, + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + }, + { + type: "response.completed", + response: { id: "resp_2", usage: {} }, + }, + ]); + + expect((output as unknown as AnyRecord).hostedSearchCitations).toBeUndefined(); + expect( + output.content.filter((b) => ((b ?? {}) as unknown as AnyRecord).type === "hostedSearch"), + ).toHaveLength(0); + expect( + output.content.find((b) => ((b ?? {}) as unknown as AnyRecord).type === "text"), + ).toMatchObject({ text: "ok" }); + }); +}); + +describe("pi-ai hosted web search: responses message replay", () => { + it("replays a hostedSearch block as a web_search_call output item", async () => { + const { convertResponsesMessages } = await import("@earendil-works/pi-ai/api/openai-responses-shared"); + const model = { + id: "gpt-test", + api: "openai-responses", + provider: "openai", + input: ["text"], + } as unknown as Parameters[1]; + const context = { + messages: [ + { + role: "user", + content: "search for the release notes", + }, + { + role: "assistant", + content: [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "pi-desktop release notes" }, + }, + }, + { type: "text", text: "Found it." }, + ], + api: "openai-responses", + provider: "openai", + model: "gpt-test", + }, + ], + } as unknown as Parameters[2]; + + const replay = convertResponsesMessages( + model as never, + context as never, + new Set(), + {}, + ) as unknown as AnyRecord[]; + + const searchItem = replay.find( + (item) => item.type === "web_search_call", + ) as unknown as AnyRecord | undefined; + expect(searchItem).toBeDefined(); + expect(searchItem?.id).toBe("ws_1"); + expect(searchItem?.status).toBe("completed"); + expect((searchItem?.action as AnyRecord)?.query).toBe( + "pi-desktop release notes", + ); + }); +}); + +describe("pi-ai hosted web search: streaming progress events", () => { + it("emits hosted_search_update as each round starts and finishes", async () => { + const { output, stream } = await processEvents([ + { type: "response.created", response: { id: "resp_1" } }, + { + type: "response.output_item.added", + output_index: 0, + item: { + type: "web_search_call", + id: "ws_1", + status: "in_progress", + action: { type: "search", query: "first query" }, + }, + }, + { + type: "response.output_item.added", + output_index: 1, + item: { + type: "web_search_call", + id: "ws_2", + status: "in_progress", + action: { type: "search", query: "second query" }, + }, + }, + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "first query" }, + }, + }, + { + type: "response.output_item.done", + output_index: 1, + item: { + type: "web_search_call", + id: "ws_2", + status: "completed", + action: { type: "search", query: "second query" }, + }, + }, + { type: "response.completed", response: { id: "resp_1", usage: {} } }, + ]); + + // Each round is its own block, in provider order. + const blocks = output.content.filter( + (block) => ((block ?? {}) as unknown as AnyRecord).type === "hostedSearch", + ) as unknown as AnyRecord[]; + expect(blocks).toHaveLength(2); + expect(blocks[0]?.blockId).toBe("ws_1"); + expect(blocks[1]?.blockId).toBe("ws_2"); + expect(blocks[0]?.status).toBe("completed"); + expect(blocks[1]?.status).toBe("completed"); + + // Two starts + two finishes: progress reaches the consumer per round. + const updates = stream.filter((event) => event.type === "hosted_search_update"); + expect(updates).toHaveLength(4); + expect(updates.map((event) => event.contentIndex)).toEqual([0, 1, 0, 1]); + }); +}); + +describe("pi-agent-core hosted web search forwarding", () => { + it("forwards hosted_search_update as message_update", async () => { + // Locks the agent-loop patch (patches/@earendil-works__pi-agent-core@0.85.1.patch): + // without it the loop's switch drops the event and search rounds render only + // after the whole turn finishes. + const { agentLoop } = await import("@earendil-works/pi-agent-core"); + + const partial = { + role: "assistant", + content: [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "in_progress", + wire: { type: "web_search_call", id: "ws_1", status: "in_progress" }, + }, + ], + api: "openai-responses", + provider: "openai", + model: "gpt-test", + stopReason: "pending", + usage: {}, + timestamp: Date.now(), + }; + const finalMessage = { + ...partial, + stopReason: "stop", + content: [ + { ...partial.content[0], status: "completed" }, + { type: "text", text: "done" }, + ], + }; + const streamEvents = [ + { type: "start", partial }, + { type: "hosted_search_update", contentIndex: 0, partial }, + { type: "done" }, + ]; + const streamFn = () => { + const iterable = (async function* () { + for (const event of streamEvents) yield event; + })(); + return Object.assign(iterable, { + result: async () => finalMessage, + }); + }; + + const emitted: AnyRecord[] = []; + const agentStream = agentLoop( + [{ role: "user", content: "search please", timestamp: Date.now() }], + { systemPrompt: "", messages: [], tools: [] }, + { + model: { + id: "gpt-test", + api: "openai-responses", + provider: "openai", + }, + convertToLlm: async (messages: unknown) => messages, + } as never, + new AbortController().signal, + streamFn as never, + ); + for await (const event of agentStream as AsyncIterable) { + emitted.push(event); + } + + const updates = emitted.filter((event) => event.type === "message_update"); + expect(updates).toHaveLength(1); + const message = updates[0]?.message as AnyRecord; + expect((message.content as AnyRecord[])[0]).toMatchObject({ + type: "hostedSearch", + blockId: "ws_1", + }); + expect(emitted.some((event) => event.type === "message_end")).toBe(true); + }); +}); + +describe("pi-ai hosted web search: request params", () => { + async function captureParams(model: AnyRecord, options: AnyRecord = {}): Promise { + const { streamSimple } = await import("@earendil-works/pi-ai/api/openai-responses"); + let payload: AnyRecord | undefined; + const sse = + "event: response.completed\n" + + 'data: {"type":"response.completed","response":{"id":"r1","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n'; + const stream = streamSimple( + { + id: "kimi-k2.8", + api: "openai-responses", + provider: "self", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + baseUrl: "http://localhost/v1", + ...model, + } as never, + { messages: [] } as never, + { + apiKey: "test", + onPayload: (params: AnyRecord) => { + payload = params; + }, + fetch: (async () => + new Response(sse, { + status: 200, + headers: { "content-type": "text/event-stream" }, + })) as never, + ...options, + } as never, + ); + const result = await stream.result(); + expect(result.stopReason).toBe("stop"); + expect(payload).toBeDefined(); + return payload!; + } + + it("attaches the search tool and asks for action sources", async () => { + const params = await captureParams({ webSearch: true }); + expect(params.tools).toEqual([{ type: "web_search" }]); + expect(params.include).toEqual(["web_search_call.action.sources"]); + }); + + it("merges the sources include with the reasoning include", async () => { + // Any explicit effort arms the reasoning branch, which assigns + // `params.include` outright — the merge must survive that. + const params = await captureParams( + { webSearch: true, thinkingLevelMap: { high: "high" } }, + { reasoning: "high" }, + ); + expect(params.include).toEqual( + expect.arrayContaining([ + "reasoning.encrypted_content", + "web_search_call.action.sources", + ]), + ); + expect((params.reasoning as AnyRecord)?.effort).toBe("high"); + }); + +}); diff --git a/packages/agent-runtime/src/model-capabilities.ts b/packages/agent-runtime/src/model-capabilities.ts index 780610fc57..dc26dfff34 100644 --- a/packages/agent-runtime/src/model-capabilities.ts +++ b/packages/agent-runtime/src/model-capabilities.ts @@ -76,6 +76,7 @@ export function modelConfigWithBinding( | "thinkingLevels" | "supportsImages" | "supportsDocuments" + | "nativeWebSearch" > | null, ): ModelConfig { @@ -110,6 +111,7 @@ export function modelConfigWithBinding( supportedThinkingLevels: enabledThinkingLevels, ...(Object.keys(thinkingLevelMap).length > 0 ? { thinkingLevelMap } : {}), ...modalityOverride(model, binding), + ...(binding.nativeWebSearch === true ? { webSearch: true } : {}), }; } diff --git a/packages/agent-runtime/src/runtime.test.ts b/packages/agent-runtime/src/runtime.test.ts index 5e60d24306..02d4b09abe 100644 --- a/packages/agent-runtime/src/runtime.test.ts +++ b/packages/agent-runtime/src/runtime.test.ts @@ -8163,3 +8163,95 @@ describe("DesktopAgentRuntime compaction summary retry and sizing (#543, ADR 028 await runtime.dispose(); }); }); + +describe("DesktopAgentRuntime hosted web search rounds (ADR 0294)", () => { + it("emits each search round as it happens and closes open rounds on message_end", async () => { + const onEvent = vi.fn(); + const runtime = createRuntime({ onEvent }); + const handleAgentEvent = (runtime as any).handleAgentEvent.bind(runtime); + + const round = (id: string, status: string, query: string, sources: unknown[] = []) => ({ + type: "hostedSearch", + phase: "web_search_call", + blockId: id, + status, + wire: { + type: "web_search_call", + id, + status, + action: { type: "search", query, ...(sources.length ? { sources } : {}) }, + }, + }); + + await handleAgentEvent({ type: "agent_start" }); + await handleAgentEvent({ type: "turn_start" }); + // Round 1 starts searching. + await handleAgentEvent({ + type: "message_start", + message: { role: "assistant", content: [round("ws_1", "in_progress", "first query")] }, + }); + // Round 1 completes and round 2 starts, still without any text. + await handleAgentEvent({ + type: "message_update", + message: { + role: "assistant", + content: [ + round("ws_1", "completed", "first query", [ + { url: "https://example.com/a", title: "A" }, + ]), + round("ws_2", "in_progress", "second query"), + ], + }, + }); + // The turn ends while round 2 is still open; text arrived meanwhile. + await handleAgentEvent({ + type: "message_end", + message: assistantMessage({ + content: [ + round("ws_1", "completed", "first query", [ + { url: "https://example.com/a", title: "A" }, + ]), + round("ws_2", "in_progress", "second query"), + { type: "text", text: "The answer." }, + ], + }), + }); + await handleAgentEvent({ type: "turn_end" }); + await handleAgentEvent({ type: "agent_end", messages: [] }); + + const events = onEvent.mock.calls.map(([envelope]) => (envelope as any).event); + const searchUpdates = events.filter( + (event: any) => event.type === "message_update" && event.message?.hostedSearch, + ); + // Round 1 streamed the moment it started; round 2 the moment it appeared. + expect(searchUpdates[0]?.message?.hostedSearch?.rounds).toEqual([ + { id: "ws_1", status: "searching", query: "first query", sources: [] }, + ]); + expect(searchUpdates.at(-1)?.message?.hostedSearch?.rounds).toEqual([ + { + id: "ws_1", + status: "completed", + query: "first query", + sources: [{ url: "https://example.com/a", title: "A" }], + }, + { id: "ws_2", status: "searching", query: "second query", sources: [] }, + ]); + + const end = events.find((event: any) => event.type === "message_end"); + // A round still open when the turn ends closes as completed, not left + // blinking "searching" in a settled transcript. + expect(end?.message?.hostedSearch).toEqual({ + status: "completed", + rounds: [ + { + id: "ws_1", + status: "completed", + query: "first query", + sources: [{ url: "https://example.com/a", title: "A" }], + }, + { id: "ws_2", status: "completed", query: "second query", sources: [] }, + ], + }); + await runtime.dispose(); + }); +}); diff --git a/packages/agent-runtime/src/runtime.ts b/packages/agent-runtime/src/runtime.ts index 565c5e70f7..be253c8106 100644 --- a/packages/agent-runtime/src/runtime.ts +++ b/packages/agent-runtime/src/runtime.ts @@ -95,6 +95,7 @@ import { DEFAULT_SUBAGENT_PERMISSION, formatAskToolOutput, formatSessionMessage, + hostedSearchFromBlocks, isCommandShellOption, isToolsOutputParams, MAX_SUBAGENT_CONCURRENCY, @@ -6592,6 +6593,33 @@ Delegation rules: } } + /** + * Fold pi-ai hostedSearch content blocks and message citations into the + * current assistant bubble as `UiMessage.hostedSearch`, emitting a + * full-frame message_update when the normalized state actually changed. + * Search state transitions are low frequency, so a full frame costs less + * than teaching every delta path about the field. + */ + private applyHostedSearch(message: unknown): void { + if (!this.currentAssistant) return; + const record = message as { + content?: unknown; + hostedSearchCitations?: unknown; + }; + const next = hostedSearchFromBlocks({ + content: record?.content, + citations: record?.hostedSearchCitations, + }); + if (!next) return; + const previous = this.currentAssistant.hostedSearch; + if (previous && JSON.stringify(previous) === JSON.stringify(next)) return; + this.currentAssistant = { + ...this.currentAssistant, + hostedSearch: next, + }; + this.emit({ type: "message_update", message: this.currentAssistant }); + } + private async handleAgentEvent(event: AgentEvent) { this.forwardAgentEventToExtensions(event); switch (event.type) { @@ -6657,6 +6685,7 @@ Delegation rules: } else { this.emit({ type: "message_start", message: this.currentAssistant }); } + this.applyHostedSearch(event.message); } // User messages are echoed and persisted by the desktop main process // (agentPrompt handler); re-emitting them here would duplicate the @@ -6665,6 +6694,7 @@ Delegation rules: } case "message_update": { if (this.currentAssistant && event.message.role === "assistant") { + this.applyHostedSearch(event.message); const content = assistantContent((event.message as any).content); const previousText = this.currentAssistant.content; const previousThinking = this.currentAssistant.thinking ?? ""; @@ -6781,6 +6811,21 @@ Delegation rules: ); } const usage = usageFromPi((event.message as any).usage as Usage | undefined); + const hostedSearch = hostedSearchFromBlocks({ + content: (event.message as any).content, + citations: (event.message as any).hostedSearchCitations, + }); + if (hostedSearch) { + const terminal = failed || aborted ? "failed" : "completed"; + for (const round of hostedSearch.rounds) { + if (round.status === "searching") round.status = terminal; + } + hostedSearch.status = hostedSearch.rounds.some( + (round) => round.status === "failed", + ) + ? "failed" + : "completed"; + } const endedAt = Date.now(); const providerWaitMs = this.requestStartedAt !== undefined && @@ -6952,6 +6997,7 @@ Delegation rules: ...(classifiedError ? { error: classifiedError, isError: true } : {}), + ...(hostedSearch ? { hostedSearch } : {}), }; this.emit({ type: "message_end", message: this.currentAssistant }); this.activeProviderRetryAttempt = 0; diff --git a/packages/agent-runtime/src/thinking-level.ts b/packages/agent-runtime/src/thinking-level.ts index 3ba6ad745d..87e18cb8b3 100644 --- a/packages/agent-runtime/src/thinking-level.ts +++ b/packages/agent-runtime/src/thinking-level.ts @@ -55,6 +55,13 @@ export type ModelConfig = { input: Array<"text" | "image">; contextWindow: number; maxTokens: number; + /** + * Opt-in for the provider-hosted web search tool. Set from the model + * binding when the user enables native web search for this model; the + * adapter attaches the vendor tool and extracts its stream blocks only + * when this is true. + */ + webSearch?: boolean; headers?: Record; compat?: Record; /** diff --git a/packages/i18n/src/locales/de/index.ts b/packages/i18n/src/locales/de/index.ts index 3f19276250..ab2ab613e3 100644 --- a/packages/i18n/src/locales/de/index.ts +++ b/packages/i18n/src/locales/de/index.ts @@ -444,6 +444,15 @@ export const de = { "modeGoal": "Ziel", "modeAgent": "Agent", "thinking": "Denken", + "webSearching": "Websuche läuft", + "webSearch": "Websuche", + "webOpenPage": "Seite öffnen", + "webFindInPage": "In Seite suchen", + "webSearchSources": "{{count}} Quelle", + "webSearchSources_other": "{{count}} Quellen", + "webSearchFailed": "Websuche fehlgeschlagen", + "webSearchMore": "+{{count}} weitere Quelle", + "webSearchMore_other": "+{{count}} weitere Quellen", "thinkingShow": "Denken anzeigen", "thinkingHide": "Denken ausblenden", "untitledTask": "Neue Aufgabe" @@ -1116,6 +1125,9 @@ sklm: { "contextWindowCatalogHint": "Folgt models.dev; eine Änderung fixiert deinen Wert.", "availableForSubagents": "Verfügbar für AI-Delegation", "availableForSubagentsHint": "Ermöglichen Sie AI, dieses Modell zu verwenden, wenn Aufgaben an Subagenten delegiert werden.", + "nativeWebSearch": "Native Websuche", + "nativeWebSearchHint": "Hängt das vom Anbieter gehostete Websuche-Tool an Anfragen für dieses Modell an. Erfordert einen Endpunkt, der dies unterstützt; die Abrechnung erfolgt pro Suche beim Anbieter.", + "nativeWebSearchUnsupported": "Die native Websuche erfordert den API-Stil „Responses“ oder „Anthropic Messages“", "searchModelId": "Modell-ID suchen…", "searchChosenModels": "Hinzugefügte Modelle suchen…", "noChosenModelMatches": "Keine passenden hinzugefügten Modelle.", diff --git a/packages/i18n/src/locales/en/index.ts b/packages/i18n/src/locales/en/index.ts index 6ee0344c07..ca85c6f683 100644 --- a/packages/i18n/src/locales/en/index.ts +++ b/packages/i18n/src/locales/en/index.ts @@ -451,6 +451,15 @@ export const en = { modeGoal: "Goal", modeAgent: "Agent", thinking: "Thinking", + webSearching: "Searching the web", + webSearch: "Web search", + webOpenPage: "Open page", + webFindInPage: "Find in page", + webSearchSources: "{{count}} source", + webSearchSources_other: "{{count}} sources", + webSearchFailed: "Web search failed", + webSearchMore: "+{{count}} more source", + webSearchMore_other: "+{{count}} more sources", thinkingShow: "Show thinking", thinkingHide: "Hide thinking", untitledTask: "New task", @@ -1137,6 +1146,9 @@ sklm: { contextWindowCatalogHint: "Follows models.dev; editing pins your value.", availableForSubagents: "Available for AI delegation", availableForSubagentsHint: "Allow AI to use this model when delegating tasks to subagents", + nativeWebSearch: "Native web search", + nativeWebSearchHint: "Attach the provider-hosted web search tool to requests for this model. Requires an endpoint that supports it; billed by the provider per search.", + nativeWebSearchUnsupported: "Native web search needs the Responses or Anthropic Messages API style", searchModelId: "Search model ID…", searchChosenModels: "Search added models…", noChosenModelMatches: "No matching added models.", diff --git a/packages/i18n/src/locales/es/index.ts b/packages/i18n/src/locales/es/index.ts index 494a237bfb..c525f876f3 100644 --- a/packages/i18n/src/locales/es/index.ts +++ b/packages/i18n/src/locales/es/index.ts @@ -444,6 +444,15 @@ export const es = { "modeGoal": "Meta", "modeAgent": "Agente", "thinking": "Pensamiento", + "webSearching": "Buscando en la web", + "webSearch": "Búsqueda web", + "webOpenPage": "Abrir página", + "webFindInPage": "Buscar en la página", + "webSearchSources": "{{count}} fuente", + "webSearchSources_other": "{{count}} fuentes", + "webSearchFailed": "Error en la búsqueda web", + "webSearchMore": "+{{count}} fuente más", + "webSearchMore_other": "+{{count}} fuentes más", "thinkingShow": "Mostrar pensamiento", "thinkingHide": "Ocultar pensamiento", "untitledTask": "Nueva tarea" @@ -1116,6 +1125,9 @@ sklm: { "contextWindowCatalogHint": "Sigue a models.dev; al editarlo se fija tu valor.", "availableForSubagents": "Disponible para delegación de AI", "availableForSubagentsHint": "Permitir que AI use este modelo al delegar tareas a subagentes", + "nativeWebSearch": "Búsqueda web nativa", + "nativeWebSearchHint": "Adjunta la herramienta de búsqueda web alojada por el proveedor a las solicitudes de este modelo. Requiere un punto de conexión que la admita; el proveedor la factura por búsqueda.", + "nativeWebSearchUnsupported": "La búsqueda web nativa requiere el estilo de API Responses o Anthropic Messages", "searchModelId": "ID de modelo de búsqueda…", "searchChosenModels": "Buscar modelos añadidos…", "noChosenModelMatches": "No hay modelos añadidos que coincidan.", diff --git a/packages/i18n/src/locales/fr/index.ts b/packages/i18n/src/locales/fr/index.ts index 5a4505e66c..ce1129df82 100644 --- a/packages/i18n/src/locales/fr/index.ts +++ b/packages/i18n/src/locales/fr/index.ts @@ -444,6 +444,15 @@ export const fr = { "modeGoal": "Objectif", "modeAgent": "Agent", "thinking": "Réflexion", + "webSearching": "Recherche web en cours", + "webSearch": "Recherche web", + "webOpenPage": "Ouvrir la page", + "webFindInPage": "Rechercher dans la page", + "webSearchSources": "{{count}} source", + "webSearchSources_other": "{{count}} sources", + "webSearchFailed": "Échec de la recherche web", + "webSearchMore": "+{{count}} autre source", + "webSearchMore_other": "+{{count}} autres sources", "thinkingShow": "Afficher la réflexion", "thinkingHide": "Masquer la réflexion", "untitledTask": "Nouvelle tâche" @@ -1116,6 +1125,9 @@ sklm: { "contextWindowCatalogHint": "Suit models.dev ; une modification fixe votre valeur.", "availableForSubagents": "Disponible pour la délégation de l'IA", "availableForSubagentsHint": "Autoriser l'IA à utiliser ce modèle lors de la délégation de tâches à des sous-agents", + "nativeWebSearch": "Recherche web native", + "nativeWebSearchHint": "Attache l'outil de recherche web hébergé par le fournisseur aux requêtes pour ce modèle. Nécessite un point de terminaison qui le prend en charge ; facturé par le fournisseur à chaque recherche.", + "nativeWebSearchUnsupported": "La recherche web native nécessite le style d'API Responses ou Anthropic Messages", "searchModelId": "ID de modèle de recherche…", "searchChosenModels": "Rechercher les modèles ajoutés…", "noChosenModelMatches": "Aucun modèle ajouté correspondant.", diff --git a/packages/i18n/src/locales/ko/index.ts b/packages/i18n/src/locales/ko/index.ts index 2545ec222f..9455653a38 100644 --- a/packages/i18n/src/locales/ko/index.ts +++ b/packages/i18n/src/locales/ko/index.ts @@ -453,6 +453,15 @@ export const ko = { modeGoal: "목표", modeAgent: "에이전트", thinking: "생각 중", + webSearching: "웹 검색 중", + webSearch: "웹 검색", + webOpenPage: "페이지 열기", + webFindInPage: "페이지에서 찾기", + webSearchSources: "출처 {{count}}개", + webSearchSources_other: "출처 {{count}}개", + webSearchFailed: "웹 검색 실패", + webSearchMore: "출처 {{count}}개 더", + webSearchMore_other: "출처 {{count}}개 더", thinkingShow: "생각 표시", thinkingHide: "생각 숨기기", untitledTask: "새 작업", @@ -1137,6 +1146,9 @@ sklm: { contextWindowCatalogHint: "models.dev를 따릅니다. 수정하면 내 값으로 고정됩니다.", availableForSubagents: "AI 위임에 사용 가능", availableForSubagentsHint: "서브에이전트에 작업을 위임할 때 AI가 이 모델을 사용하도록 허용", + nativeWebSearch: "네이티브 웹 검색", + nativeWebSearchHint: "이 모델에 대한 요청에 공급자 호스팅 웹 검색 도구를 첨부합니다. 이를 지원하는 엔드포인트가 필요하며, 검색 건수로 공급자에게 과금됩니다.", + nativeWebSearchUnsupported: "네이티브 웹 검색에는 Responses 또는 Anthropic Messages API 스타일이 필요합니다", searchModelId: "모델 ID 검색…", searchChosenModels: "추가된 모델 검색…", noChosenModelMatches: "일치하는 추가된 모델이 없습니다.", diff --git a/packages/i18n/src/locales/tr/index.ts b/packages/i18n/src/locales/tr/index.ts index a307e7758b..48222eb23c 100644 --- a/packages/i18n/src/locales/tr/index.ts +++ b/packages/i18n/src/locales/tr/index.ts @@ -453,6 +453,15 @@ export const tr = { modeGoal: "Hedef", modeAgent: "Ajan", thinking: "Düşünme", + webSearching: "Web araması sürüyor", + webSearch: "Web araması", + webOpenPage: "Sayfayı aç", + webFindInPage: "Sayfada bul", + webSearchSources: "{{count}} kaynak", + webSearchSources_other: "{{count}} kaynak", + webSearchFailed: "Web araması başarısız", + webSearchMore: "+{{count}} kaynak daha", + webSearchMore_other: "+{{count}} kaynak daha", thinkingShow: "Düşünmeyi göster", thinkingHide: "Düşünmeyi gizle", untitledTask: "Yeni görev", @@ -1137,6 +1146,9 @@ sklm: { contextWindowCatalogHint: "models.dev'i takip eder; düzenlerseniz değeriniz sabitlenir.", availableForSubagents: "AI devri için kullanılabilir", availableForSubagentsHint: "Görevler alt ajanlara devredilirken AI’nin bu modeli kullanmasına izin ver", + nativeWebSearch: "Yerel web araması", + nativeWebSearchHint: "Bu modele yapılan isteklere sağlayıcının barındırdığı web arama aracını ekler. Bunu destekleyen bir uç nokta gerekir; arama başına sağlayıcı tarafından faturalandırılır.", + nativeWebSearchUnsupported: "Yerel web araması için Responses veya Anthropic Messages API stili gerekir", searchModelId: "Model kimliği ara…", searchChosenModels: "Eklenen modellerde ara…", noChosenModelMatches: "Eşleşen eklenen model yok.", diff --git a/packages/i18n/src/locales/zh-CN/index.ts b/packages/i18n/src/locales/zh-CN/index.ts index 46715a566d..4ee9b73d63 100644 --- a/packages/i18n/src/locales/zh-CN/index.ts +++ b/packages/i18n/src/locales/zh-CN/index.ts @@ -448,6 +448,15 @@ export const zhCN = { modeGoal: "目标", modeAgent: "智能体", thinking: "思考", + webSearching: "正在联网搜索", + webSearch: "联网搜索", + webOpenPage: "打开网页", + webFindInPage: "页内查找", + webSearchSources: "{{count}} 个来源", + webSearchSources_other: "{{count}} 个来源", + webSearchFailed: "联网搜索失败", + webSearchMore: "还有 {{count}} 个来源", + webSearchMore_other: "还有 {{count}} 个来源", thinkingShow: "显示思考过程", thinkingHide: "隐藏思考过程", untitledTask: "新建任务", @@ -1120,6 +1129,9 @@ sklm: { contextWindowCatalogHint: "跟随 models.dev;手动修改后会固定为你的值", availableForSubagents: "可供 AI 自动调度", availableForSubagentsHint: "允许 AI 在委派子任务时自动选用此模型", + nativeWebSearch: "原生联网搜索", + nativeWebSearchHint: "为该模型的请求附加提供商托管的联网搜索工具。需要端点实际支持;按次计费由提供商收取。", + nativeWebSearchUnsupported: "原生联网搜索需要 Responses 或 Anthropic Messages 接口风格", searchModelId: "搜索模型 ID…", searchChosenModels: "搜索已添加模型…", noChosenModelMatches: "没有匹配的已添加模型", diff --git a/packages/i18n/src/locales/zh-TW/index.ts b/packages/i18n/src/locales/zh-TW/index.ts index 21e9f4c749..700ee7017f 100644 --- a/packages/i18n/src/locales/zh-TW/index.ts +++ b/packages/i18n/src/locales/zh-TW/index.ts @@ -448,6 +448,15 @@ export const zhTW = { modeGoal: "目標", modeAgent: "智慧體", thinking: "思考", + webSearching: "正在網路搜尋", + webSearch: "網路搜尋", + webOpenPage: "開啟網頁", + webFindInPage: "頁內尋找", + webSearchSources: "{{count}} 個來源", + webSearchSources_other: "{{count}} 個來源", + webSearchFailed: "網路搜尋失敗", + webSearchMore: "還有 {{count}} 個來源", + webSearchMore_other: "還有 {{count}} 個來源", thinkingShow: "顯示思考過程", thinkingHide: "隱藏思考過程", untitledTask: "新建任務", @@ -1120,6 +1129,9 @@ sklm: { contextWindowCatalogHint: "跟隨 models.dev;手動修改後會固定為你的值", availableForSubagents: "可供 AI 自動排程", availableForSubagentsHint: "允許 AI 在委派子任務時自動選用此模型", + nativeWebSearch: "原生網路搜尋", + nativeWebSearchHint: "為該模型的請求附加提供商託管的網路搜尋工具。需要端點實際支援;按次計費由提供商收取。", + nativeWebSearchUnsupported: "原生網路搜尋需要 Responses 或 Anthropic Messages 介面風格", searchModelId: "搜尋模型 ID…", searchChosenModels: "搜尋已新增模型…", noChosenModelMatches: "沒有符合的已新增模型", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index e1b1f7e844..d1e1167982 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -57,3 +57,4 @@ export * from "./message-stream.js"; export * from "./session-collaboration.js"; export * from "./window-chrome.js"; export * from "./prompt-enhancement.js"; +export * from "./native-web-search.js"; diff --git a/packages/shared/src/message-stream.ts b/packages/shared/src/message-stream.ts index b96f810732..e320e3547d 100644 --- a/packages/shared/src/message-stream.ts +++ b/packages/shared/src/message-stream.ts @@ -44,6 +44,7 @@ export function streamingMessageIdentity(message: UiMessage): UiMessage { ? { parentToolCallId: message.parentToolCallId } : {}), ...(message.agentName ? { agentName: message.agentName } : {}), + ...(message.hostedSearch ? { hostedSearch: message.hostedSearch } : {}), }; } @@ -172,6 +173,11 @@ export function applyMessageUpdate( ? { parentToolCallId: event.message.parentToolCallId } : {}), ...(event.message.agentName ? { agentName: event.message.agentName } : {}), + ...(event.message.hostedSearch + ? { hostedSearch: event.message.hostedSearch } + : seed.hostedSearch + ? { hostedSearch: seed.hostedSearch } + : {}), }; } diff --git a/packages/shared/src/native-web-search.test.ts b/packages/shared/src/native-web-search.test.ts new file mode 100644 index 0000000000..c918dc6507 --- /dev/null +++ b/packages/shared/src/native-web-search.test.ts @@ -0,0 +1,465 @@ +import { describe, expect, it } from "vitest"; + +import { + hostedSearchFromBlocks, + hostedSearchRounds, + nativeWebSearchToolFor, + resolveNativeWebSearch, +} from "./native-web-search.js"; + +describe("resolveNativeWebSearch", () => { + it("turns on only for wires that define a hosted search tool", () => { + expect( + resolveNativeWebSearch({ wireApi: "anthropic-messages", modelWebSearch: true }), + ).toBe("on"); + expect( + resolveNativeWebSearch({ wireApi: "openai-responses", modelWebSearch: true }), + ).toBe("on"); + expect( + resolveNativeWebSearch({ wireApi: "azure-openai-responses", modelWebSearch: true }), + ).toBe("on"); + }); + + it("stays off for wires without a hosted search tool", () => { + for (const wire of [ + "openai-completions", + "openai-codex-responses", + "google-generative-ai", + "pi-messages", + "", + ]) { + expect(resolveNativeWebSearch({ wireApi: wire, modelWebSearch: true })).toBe("off"); + } + }); + + it("requires the explicit binding opt-in; absence is off", () => { + expect(resolveNativeWebSearch({ wireApi: "openai-responses" })).toBe("off"); + expect( + resolveNativeWebSearch({ wireApi: "openai-responses", modelWebSearch: false }), + ).toBe("off"); + expect( + resolveNativeWebSearch({ wireApi: "openai-responses", modelWebSearch: undefined }), + ).toBe("off"); + }); + + it("normalizes wire spelling before matching", () => { + expect( + resolveNativeWebSearch({ wireApi: " Anthropic-Messages ", modelWebSearch: true }), + ).toBe("on"); + }); + + it("never infers support from unrelated inputs", () => { + // A gateway on a chat-completions wire stays off no matter what the + // binding says: the wire cannot carry the tool. + expect( + resolveNativeWebSearch({ wireApi: "openai-completions", modelWebSearch: true }), + ).toBe("off"); + }); +}); + +describe("nativeWebSearchToolFor", () => { + it("maps each supported wire to its vendor tool shape", () => { + expect(nativeWebSearchToolFor("anthropic-messages")).toEqual({ + type: "web_search_20250305", + name: "web_search", + }); + expect(nativeWebSearchToolFor("openai-responses")).toEqual({ type: "web_search" }); + expect(nativeWebSearchToolFor("azure-openai-responses")).toEqual({ + type: "web_search", + }); + }); + + it("returns undefined for wires without a tool shape", () => { + expect(nativeWebSearchToolFor("openai-completions")).toBeUndefined(); + expect(nativeWebSearchToolFor("")).toBeUndefined(); + }); +}); + +describe("hostedSearchFromBlocks", () => { + it("normalizes a responses web_search_call block into one round", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "pi-desktop release notes" }, + results: [ + { url: "https://example.com/a", title: "A" }, + { url: "https://example.com/b", title: " " }, + ], + }, + }, + ], + citations: [ + { url: "https://example.com/a", title: "A" }, + { url: "https://example.com/c", title: "C" }, + { url: "not-a-url" }, + ], + }); + expect(search).toEqual({ + status: "completed", + rounds: [ + { + id: "ws_1", + status: "completed", + query: "pi-desktop release notes", + sources: [ + { url: "https://example.com/a", title: "A" }, + { url: "https://example.com/b" }, + // Citation-only URLs fold into the most recent round, deduped. + { url: "https://example.com/c", title: "C" }, + ], + }, + ], + }); + }); + + it("keeps each search round separate, in provider order", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { + type: "search", + query: "first query", + sources: [{ url: "https://example.com/1" }], + }, + }, + }, + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_2", + status: "in_progress", + wire: { + type: "web_search_call", + id: "ws_2", + status: "in_progress", + action: { type: "search", query: "second query" }, + }, + }, + ], + }); + expect(search).toEqual({ + status: "searching", + rounds: [ + { + id: "ws_1", + status: "completed", + query: "first query", + sources: [{ url: "https://example.com/1" }], + }, + { id: "ws_2", status: "searching", query: "second query", sources: [] }, + ], + }); + }); + + it("pairs an anthropic server_tool_use with its result into one round", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_1", + name: "web_search", + input: { query: "rust async" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvu_1", + wire: { + type: "web_search_tool_result", + tool_use_id: "srvu_1", + content: [ + { + type: "web_search_result", + url: "https://example.com/rust", + title: "Rust async", + }, + ], + }, + }, + ], + }); + expect(search).toEqual({ + status: "completed", + rounds: [ + { + id: "srvu_1", + status: "completed", + query: "rust async", + sources: [{ url: "https://example.com/rust", title: "Rust async" }], + }, + ], + }); + }); + + it("marks an anthropic error result as a failed round", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_1", + input: { query: "rust async" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvu_1", + isError: true, + wire: { + type: "web_search_tool_result_error", + tool_use_id: "srvu_1", + content: "search unavailable", + }, + }, + ], + }); + expect(search).toEqual({ + status: "failed", + rounds: [ + { id: "srvu_1", status: "failed", query: "rust async", sources: [] }, + ], + }); + }); + + it("pairs an id-less result with the latest round still searching", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_1", + input: { query: "done round" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvu_1", + wire: { type: "web_search_tool_result", content: [] }, + }, + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_2", + input: { query: "open round" }, + }, + // A gateway that drops the tool_use_id still lands the result on the + // round that is actually open instead of inventing a new one. + { + type: "hostedSearch", + phase: "web_search_tool_result", + wire: { + type: "web_search_tool_result", + content: [{ url: "https://example.com/open" }], + }, + }, + ], + }); + expect(search?.rounds).toEqual([ + { id: "srvu_1", status: "completed", query: "done round", sources: [] }, + { + id: "srvu_2", + status: "completed", + query: "open round", + sources: [{ url: "https://example.com/open" }], + }, + ]); + }); + + it("labels responses open_page and find_in_page rounds with their target", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "openclaw latest" }, + }, + }, + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_2", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_2", + status: "completed", + action: { type: "open_page", url: "https://www.npmjs.com/package/openclaw" }, + }, + }, + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_3", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_3", + status: "completed", + action: { + type: "find_in_page", + url: "https://docs.openclaw.ai/releases", + pattern: "2026.9", + }, + }, + }, + ], + }); + expect(search?.rounds).toEqual([ + { id: "ws_1", status: "completed", query: "openclaw latest", sources: [] }, + { + id: "ws_2", + status: "completed", + kind: "openPage", + url: "https://www.npmjs.com/package/openclaw", + sources: [], + }, + { + id: "ws_3", + status: "completed", + kind: "findInPage", + url: "https://docs.openclaw.ai/releases", + query: "2026.9", + sources: [], + }, + ]); + }); + + it("labels an anthropic web_fetch use as an open-page round", () => { + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_9", + name: "web_fetch", + input: { url: "https://example.com/page" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvu_9", + wire: { type: "web_search_tool_result", content: [] }, + }, + ], + }); + expect(search?.rounds).toEqual([ + { + id: "srvu_9", + status: "completed", + kind: "openPage", + url: "https://example.com/page", + sources: [], + }, + ]); + }); + + it("reads the query from gateway-native search field names", () => { + // GLM's web_search_prime (relayed by gateways onto the anthropic wire) + // names its input field `search_query`, not `query`. + const search = hostedSearchFromBlocks({ + content: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvu_glm", + name: "web_search_prime", + input: { location: "us", search_query: "OpenClaw latest version release" }, + }, + ], + }); + expect(search?.rounds).toEqual([ + { + id: "srvu_glm", + status: "searching", + query: "OpenClaw latest version release", + sources: [], + }, + ]); + }); + + it("returns undefined without hostedSearch blocks and never throws on junk", () => { + expect(hostedSearchFromBlocks({ content: [] })).toBeUndefined(); + expect( + hostedSearchFromBlocks({ content: [{ type: "text", text: "hi" }] }), + ).toBeUndefined(); + // A block without a recognized phase carries no round information. + expect( + hostedSearchFromBlocks({ + content: [null, 7, { type: "hostedSearch" }] as unknown[], + }), + ).toBeUndefined(); + expect( + hostedSearchFromBlocks({ + content: [{ type: "hostedSearch", phase: "web_search_call", wire: "junk" }], + citations: [{ url: "ftp://nope" }], + }), + ).toEqual({ + status: "searching", + rounds: [{ id: "anon-0", status: "searching", sources: [] }], + }); + }); +}); + +describe("hostedSearchRounds", () => { + it("passes through the per-round shape", () => { + const rounds = [ + { id: "ws_1", status: "completed" as const, query: "q", sources: [] }, + ]; + expect(hostedSearchRounds({ status: "completed", rounds })).toEqual(rounds); + }); + + it("collapses a v1 aggregate transcript into one legacy round", () => { + const legacy = { + status: "completed", + queries: ["old query"], + sources: [{ url: "https://example.com/a", title: "A" }], + }; + expect( + hostedSearchRounds(legacy as unknown as Parameters[0]), + ).toEqual([ + { + id: "legacy", + status: "completed", + query: "old query", + sources: [{ url: "https://example.com/a", title: "A" }], + }, + ]); + }); + + it("reads nothing from empty or missing data", () => { + expect(hostedSearchRounds(undefined)).toEqual([]); + expect( + hostedSearchRounds({ status: "completed", rounds: [] }), + ).toEqual([]); + expect( + hostedSearchRounds({ + status: "completed", + queries: [], + sources: [], + } as unknown as Parameters[0]), + ).toEqual([]); + }); +}); diff --git a/packages/shared/src/native-web-search.ts b/packages/shared/src/native-web-search.ts new file mode 100644 index 0000000000..53e9c53eb8 --- /dev/null +++ b/packages/shared/src/native-web-search.ts @@ -0,0 +1,332 @@ +/** + * Single evaluation point for the provider-hosted web search tool. + * + * Every consumer — the request assembly in the pi-ai adapters, the runtime, + * and any UI gating — must resolve "does this request carry the hosted web + * search tool" through `resolveNativeWebSearch` and nothing else. Detection + * inputs are deliberately minimal: + * + * - `wireApi` is the RESOLVED wire API for the model (the result of + * `apiBindingForProviderModel`), never the provider's stored apiStyle: a + * model-level catalog pin can select a different wire than the provider + * row, and judging by the stored style produced UI/runtime disagreement + * before. + * - `modelWebSearch` is the `ModelConfig.webSearch` flag derived from the + * user's per-model binding opt-in. models.dev publishes no hosted-tool + * capability, so there is no catalog default: only an explicit user + * opt-in can enable the tool. + * + * Vendor display names, base URL hostnames, and model id substrings are + * intentionally not consulted. An endpoint either carries the tool on the + * wire named here or it does not; guessing breeds silent behavior drift. + */ + +/** Wire APIs whose request format defines a provider-hosted search tool. */ +export const NATIVE_WEB_SEARCH_WIRE_APIS = new Set([ + "anthropic-messages", + "openai-responses", + "azure-openai-responses", +]); + +/** Tool definition attached to an anthropic-messages request. */ +export const ANTHROPIC_WEB_SEARCH_TOOL = { + type: "web_search_20250305", + name: "web_search", +} as const; + +/** Tool definition attached to an openai-responses request. */ +export const OPENAI_RESPONSES_WEB_SEARCH_TOOL = { + type: "web_search", +} as const; + +export type NativeWebSearchDecision = "on" | "off"; + +export function resolveNativeWebSearch(input: { + wireApi: string; + modelWebSearch?: boolean; +}): NativeWebSearchDecision { + const wire = input.wireApi.trim().toLowerCase(); + if (!NATIVE_WEB_SEARCH_WIRE_APIS.has(wire)) return "off"; + return input.modelWebSearch === true ? "on" : "off"; +} + +/** The tool definition a wire API expects, or undefined when unsupported. */ +export function nativeWebSearchToolFor( + wireApi: string, +): + | { type: "web_search_20250305"; name: "web_search" } + | { type: "web_search" } + | undefined { + const wire = wireApi.trim().toLowerCase(); + if (wire === "anthropic-messages") return { ...ANTHROPIC_WEB_SEARCH_TOOL }; + if (wire === "openai-responses" || wire === "azure-openai-responses") { + return { ...OPENAI_RESPONSES_WEB_SEARCH_TOOL }; + } + return undefined; +} + +/** + * Normalize pi-ai hostedSearch content blocks and message citations into the + * shared `HostedSearch` shape: one round per provider search call, in block + * order. Input is untrusted provider data: every field is shape-checked and + * unknown shapes are ignored, never thrown. + * + * Round pairing: + * - anthropic-messages emits a `server_tool_use` block (query) followed by a + * `web_search_tool_result` block (sources); both carry the same blockId. + * - openai-responses emits one `web_search_call` block per call; status and + * the action payload live on the preserved wire item. + * Citation annotations belong to the message, not a round; citation-only URLs + * fold into the most recent round so they still render exactly once. + */ +export function hostedSearchFromBlocks(input: { + content: unknown; + citations?: unknown; +}): import("./types/messages.js").HostedSearch | undefined { + const blocks = Array.isArray(input.content) ? input.content : []; + const rounds: import("./types/messages.js").HostedSearchRound[] = []; + const byId = new Map(); + + const roundFor = ( + id: unknown, + fallback: string, + create: boolean, + ): import("./types/messages.js").HostedSearchRound | undefined => { + const key = typeof id === "string" && id ? id : fallback; + const existing = byId.get(key); + if (existing) return existing; + if (!create) return undefined; + const round: import("./types/messages.js").HostedSearchRound = { + id: key, + status: "searching", + sources: [], + }; + byId.set(key, round); + rounds.push(round); + return round; + }; + + for (let index = 0; index < blocks.length; index++) { + const part = blocks[index]; + if (!part || typeof part !== "object") continue; + const block = part as { + type?: string; + phase?: string; + blockId?: unknown; + name?: unknown; + isError?: boolean; + status?: string; + input?: unknown; + wire?: unknown; + }; + if (block.type !== "hostedSearch") continue; + const wire = + block.wire && typeof block.wire === "object" + ? (block.wire as Record) + : undefined; + + if (block.phase === "server_tool_use") { + // The round stays "searching" until its result block lands. + const round = roundFor(block.blockId, `anon-${index}`, true); + if (!round) continue; + // web_fetch is the Anthropic open-page call: it carries a url, no query. + if (block.name === "web_fetch") { + round.kind = "openPage"; + const inputUrl = (value: unknown) => + value && typeof value === "object" + ? hostedSearchUrl((value as Record).url) + : undefined; + const url = inputUrl(block.input) ?? inputUrl(wire?.input); + if (url && !round.url) round.url = url; + continue; + } + const query = hostedSearchQuery(wire, block.phase, block.input); + if (query && !round.query) round.query = query; + continue; + } + + if (block.phase === "web_search_tool_result") { + // A result whose id matches no call pairs with the latest open round; + // a gateway that drops ids still renders one row per round. + const round = + roundFor(block.blockId, "", false) ?? + [...rounds].reverse().find((r) => r.status === "searching") ?? + roundFor(undefined, `anon-${index}`, true); + if (!round) continue; + round.status = block.isError === true || block.status === "failed" + ? "failed" + : "completed"; + collectHostedSearchSources(wire, round.sources); + continue; + } + + if (block.phase === "web_search_call") { + const round = roundFor(block.blockId, `anon-${index}`, true); + if (!round) continue; + const status = typeof block.status === "string" ? block.status : wireStatus(wire); + round.status = + status === "completed" + ? "completed" + : status === "failed" || block.isError === true + ? "failed" + : "searching"; + // Responses actions: `search` carries a query, `open_page` a url, + // `find_in_page` both. Items can arrive action-less mid-stream, so the + // kind lands whenever the action finally does. + const action = + wire && typeof wire.action === "object" && wire.action !== null + ? (wire.action as Record) + : undefined; + if (action?.type === "open_page") round.kind = "openPage"; + else if (action?.type === "find_in_page") round.kind = "findInPage"; + const url = hostedSearchUrl(action?.url); + if (url && !round.url && (round.kind === "openPage" || round.kind === "findInPage")) { + round.url = url; + } + const query = hostedSearchQuery(wire, block.phase, block.input); + if (query && !round.query) round.query = query; + collectHostedSearchSources(wire, round.sources); + continue; + } + } + + if (rounds.length === 0) return undefined; + + const citations = Array.isArray(input.citations) ? input.citations : []; + const last = rounds[rounds.length - 1]; + for (const citation of citations) { + if (!citation || typeof citation !== "object") continue; + const c = citation as { url?: unknown; title?: unknown }; + if (typeof c.url !== "string" || !/^https?:\/\//i.test(c.url)) continue; + if (rounds.some((r) => r.sources.some((s) => s.url === c.url))) continue; + last.sources.push({ + url: c.url, + ...(typeof c.title === "string" && c.title.trim() ? { title: c.title } : {}), + }); + } + + return { + status: rounds.some((r) => r.status === "failed") + ? "failed" + : rounds.some((r) => r.status === "searching") + ? "searching" + : "completed", + rounds, + }; +} + +/** + * Read the rounds of a persisted/streamed `HostedSearch`, tolerating the v1 + * aggregate shape (`queries`/`sources`, no rounds) written by development + * builds before per-round rows existed. Such transcripts collapse to a single + * legacy round instead of losing the row entirely. + */ +export function hostedSearchRounds( + search: import("./types/messages.js").HostedSearch | undefined, +): import("./types/messages.js").HostedSearchRound[] { + if (!search || typeof search !== "object") return []; + const rounds = (search as { rounds?: unknown }).rounds; + if (Array.isArray(rounds)) { + return rounds.filter( + (r): r is import("./types/messages.js").HostedSearchRound => + !!r && typeof r === "object", + ); + } + const legacy = search as { queries?: unknown; sources?: unknown }; + const queries = Array.isArray(legacy.queries) ? legacy.queries : []; + const sources = Array.isArray(legacy.sources) ? legacy.sources : []; + if (queries.length === 0 && sources.length === 0) return []; + return [ + { + id: "legacy", + status: search.status === "failed" ? "failed" : "completed", + ...(typeof queries[0] === "string" && queries[0] ? { query: queries[0] } : {}), + sources: sources.filter( + (s): s is import("./types/messages.js").HostedSearchSource => + !!s && typeof s === "object" && typeof (s as { url?: unknown }).url === "string", + ), + }, + ]; +} + +function hostedSearchUrl(value: unknown): string | undefined { + return typeof value === "string" && /^https?:\/\//i.test(value) + ? value + : undefined; +} + +function wireStatus(wire: Record | undefined): string | undefined { + if (!wire) return undefined; + return typeof wire.status === "string" ? wire.status : undefined; +} + +function hostedSearchQuery( + wire: Record | undefined, + phase: string | undefined, + input: unknown, +): string | undefined { + // `query` is the Anthropic/OpenAI field; gateways that relay to a native + // search tool (GLM web_search_prime et al.) may name it `search_query`. + const inputRecord = + input && typeof input === "object" + ? (input as Record) + : undefined; + const fromInput = inputRecord?.query ?? inputRecord?.search_query; + if (typeof fromInput === "string" && fromInput.trim()) { + return fromInput.trim(); + } + if (!wire) return undefined; + const action = wire.action as Record | undefined; + const query = + action && typeof action === "object" && typeof action.query === "string" + ? action.query + : action && typeof action === "object" && typeof action.pattern === "string" + ? action.pattern + : typeof wire.query === "string" + ? wire.query + : undefined; + if (query && query.trim()) return query.trim(); + // Anthropic server_tool_use input carries the query. + const wireInput = wire.input as Record | undefined; + if (phase === "server_tool_use" && wireInput) { + const wireQuery = wireInput.query ?? wireInput.search_query; + if (typeof wireQuery === "string") return wireQuery.trim() || undefined; + } + return undefined; +} + +function collectHostedSearchSources( + wire: Record | undefined, + sources: { url: string; title?: string }[], +): void { + if (!wire) return; + const action = wire.action as Record | undefined; + const candidates = [ + ...(Array.isArray(wire.results) ? wire.results : []), + ...(Array.isArray(wire.sources) ? wire.sources : []), + ...(Array.isArray(wire.search_results) ? wire.search_results : []), + // Anthropic web_search_tool_result carries web_search_result entries. + ...(Array.isArray(wire.content) ? wire.content : []), + // OpenAI Responses nests sources under the search action. + ...(action && Array.isArray(action.sources) ? action.sources : []), + ...(action && Array.isArray(action.results) ? action.results : []), + ]; + for (const candidate of candidates) { + if (!candidate || typeof candidate !== "object") continue; + const entry = candidate as { url?: unknown; uri?: unknown; title?: unknown }; + const url = + typeof entry.url === "string" + ? entry.url + : typeof entry.uri === "string" + ? entry.uri + : undefined; + if (!url || !/^https?:\/\//i.test(url)) continue; + if (sources.some((s) => s.url === url)) continue; + sources.push({ + url, + ...(typeof entry.title === "string" && entry.title.trim() + ? { title: entry.title } + : {}), + }); + } +} diff --git a/packages/shared/src/types/messages.ts b/packages/shared/src/types/messages.ts index 72bd67b4c8..3ce6391782 100644 --- a/packages/shared/src/types/messages.ts +++ b/packages/shared/src/types/messages.ts @@ -121,6 +121,49 @@ export type UiMessage = { parentToolCallId?: string; /** Definition name of the subagent that produced this row. */ agentName?: string; + /** + * Provider-hosted web search activity for this assistant turn, extracted + * from the vendor stream by the pi-ai adapters. Present only when the + * model binding opted into native web search and the provider actually + * searched. + */ + hostedSearch?: HostedSearch; +}; + +/** + * Normalized provider-hosted web search activity for one assistant message. + * One message can run several search rounds server-side; each round is one + * provider call (`server_tool_use` pair / `web_search_call` item) and renders + * as its own transcript row. + */ +export type HostedSearch = { + /** Aggregate of `rounds`: failed if any round failed, else searching while + * any round is still in flight. */ + status: "searching" | "completed" | "failed"; + rounds: HostedSearchRound[]; +}; + +/** One provider search round, in the order the provider issued it. */ +export type HostedSearchRound = { + /** Stable key within the message: the provider block/item id when known. */ + id: string; + status: "searching" | "completed" | "failed"; + /** + * What the provider did this round. Responses `web_search_call` actions map + * `search`/`open_page`/`find_in_page` onto these; an Anthropic + * `web_fetch` server tool use is an `openPage`. Absent means a plain search. + */ + kind?: "search" | "openPage" | "findInPage"; + /** The search query (or the in-page pattern for `findInPage`). */ + query?: string; + /** The page a round opened, for `openPage`/`findInPage`. */ + url?: string; + sources: HostedSearchSource[]; +}; + +export type HostedSearchSource = { + url: string; + title?: string; }; /** Terminal outcome of one background subagent run. TaskStop adds its own diff --git a/packages/shared/src/types/models.ts b/packages/shared/src/types/models.ts index ff8710ce8e..f2ff14feab 100644 --- a/packages/shared/src/types/models.ts +++ b/packages/shared/src/types/models.ts @@ -110,6 +110,13 @@ export type ModelBinding = { * parent agent can pick it at Task time. Defaults to false (opt-in). */ availableForSubagents?: boolean; + /** + * Opt-in for attaching the provider-hosted web search tool to requests for + * this model. Absent/false keeps the tool off. There is no catalog default: + * models.dev does not publish hosted-tool capability, so the user's own + * knowledge of the endpoint is the only source. + */ + nativeWebSearch?: boolean; }; export const MODEL_MODALITIES = ["text", "image", "audio", "video", "pdf"] as const; diff --git a/patches/@earendil-works__pi-agent-core@0.85.1.patch b/patches/@earendil-works__pi-agent-core@0.85.1.patch new file mode 100644 index 0000000000..56483a429b --- /dev/null +++ b/patches/@earendil-works__pi-agent-core@0.85.1.patch @@ -0,0 +1,17 @@ +diff --git a/dist/agent-loop.js b/dist/agent-loop.js +index ca1186cb920c1da692381694538b388af0c5261c..234aab333cafb376131fd364086973641ca5be1a 100644 +--- a/dist/agent-loop.js ++++ b/dist/agent-loop.js +@@ -213,6 +213,12 @@ async function streamAssistantResponse(context, config, signal, emit, streamFunc + case "toolcall_start": + case "toolcall_delta": + case "toolcall_end": ++ // Hosted web search (PI-Desktop): the pi-ai adapters push ++ // `hosted_search_update` when a provider-side search block starts ++ // or completes. Forward it like any other progress event so the ++ // application renders each search round as it happens instead of ++ // only learning about searches from the final message. ++ case "hosted_search_update": + if (partialMessage) { + partialMessage = event.partial; + context.messages[context.messages.length - 1] = partialMessage; diff --git a/patches/@earendil-works__pi-ai@0.85.1.patch b/patches/@earendil-works__pi-ai@0.85.1.patch index b110a31cde..ff7da6a1d3 100644 --- a/patches/@earendil-works__pi-ai@0.85.1.patch +++ b/patches/@earendil-works__pi-ai@0.85.1.patch @@ -1,3 +1,173 @@ +diff --git a/dist/api/anthropic-messages.js b/dist/api/anthropic-messages.js +index e1538a5dfcf2225da9dc3ec622576564ee7c8530..711636f5b847d3db15384063f30532222784395f 100644 +--- a/dist/api/anthropic-messages.js ++++ b/dist/api/anthropic-messages.js +@@ -467,6 +467,39 @@ export const stream = (model, context, options) => { + output.content.push(block); + stream.push({ type: "toolcall_start", contentIndex: output.content.length - 1, partial: output }); + } ++ // Hosted web search (PI-Desktop): server-side search blocks ++ // are captured whole. The paired `server_tool_use` + ++ // `web_search_tool_result` blocks round-trip verbatim via ++ // the hostedSearch block so multi-turn conversations keep ++ // their grounding (encrypted content included). ++ else if (event.content_block.type === "server_tool_use") { ++ const block = { ++ type: "hostedSearch", ++ phase: "server_tool_use", ++ blockId: event.content_block.id, ++ name: event.content_block.name, ++ input: event.content_block.input, ++ inputJson: "", ++ index: event.index, ++ }; ++ output.content.push(block); ++ stream.push({ type: "hosted_search_update", contentIndex: output.content.length - 1, partial: output }); ++ } ++ else if (event.content_block.type === "web_search_tool_result" || ++ event.content_block.type === "web_search_tool_result_error") { ++ const isError = event.content_block.type === "web_search_tool_result_error"; ++ const block = { ++ type: "hostedSearch", ++ phase: "web_search_tool_result", ++ blockId: event.content_block.tool_use_id, ++ isError, ++ // The raw block is the replay payload; keep it verbatim. ++ wire: event.content_block, ++ index: event.index, ++ }; ++ output.content.push(block); ++ stream.push({ type: "hosted_search_update", contentIndex: output.content.length - 1, partial: output }); ++ } + } + else if (event.type === "content_block_delta") { + if (event.delta.type === "text_delta") { +@@ -508,6 +541,18 @@ export const stream = (model, context, options) => { + partial: output, + }); + } ++ // Hosted web search (PI-Desktop): server_tool_use input ++ // (the query) streams as input_json_delta like any tool ++ // call; accumulate it so the row can show the query live. ++ else if (block && block.type === "hostedSearch" && block.phase === "server_tool_use") { ++ block.inputJson = (block.inputJson ?? "") + event.delta.partial_json; ++ block.input = parseStreamingJson(block.inputJson); ++ stream.push({ ++ type: "hosted_search_update", ++ contentIndex: index, ++ partial: output, ++ }); ++ } + } + else if (event.delta.type === "signature_delta") { + const index = blocks.findIndex((b) => b.index === event.index); +@@ -517,6 +562,16 @@ export const stream = (model, context, options) => { + block.thinkingSignature += event.delta.signature; + } + } ++ // Hosted web search (PI-Desktop): citations arrive as deltas ++ // on the text block that follows the search result. Collect ++ // them on the message for the application to render. ++ else if (event.delta.type === "citations_delta") { ++ const citation = event.delta.citation; ++ if (citation && typeof citation === "object") { ++ output.hostedSearchCitations = output.hostedSearchCitations ?? []; ++ output.hostedSearchCitations.push(citation); ++ } ++ } + } + else if (event.type === "content_block_stop") { + const index = blocks.findIndex((b) => b.index === event.index); +@@ -551,6 +606,12 @@ export const stream = (model, context, options) => { + partial: output, + }); + } ++ // Hosted search blocks carry no streaming scratch ++ // state; closing one just drops the index marker. ++ else if (block.type === "hostedSearch") { ++ delete block.index; ++ delete block.inputJson; ++ } + } + } + else if (event.type === "message_delta") { +@@ -838,6 +899,15 @@ function buildParams(model, context, isOAuthToken, options) { + ...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, compat.supportsStrictTools, undefined, true), + ]; + } ++ // Hosted web search (PI-Desktop): attach the server-side tool when the ++ // model record opts in. A hosted tool has no client execute; its blocks ++ // are extracted by the stream loop below and replayed by convertMessages. ++ if (model.webSearch === true) { ++ params.tools = [ ++ ...(params.tools ?? []), ++ { type: "web_search_20250305", name: "web_search" }, ++ ]; ++ } + // Managed effort models always use adaptive thinking so prefix mismatches can + // be dropped instead of surfacing as persistent 400 responses. + if (model.compat?.supportsMidConvoEffort === true) { +@@ -1028,6 +1098,30 @@ function convertMessages(transformedMessages, isOAuthToken, cacheControl, allowE + input: block.arguments ?? {}, + }); + } ++ // Hosted web search (PI-Desktop): replay both halves of the ++ // pair in place, in their original order. `server_tool_use` ++ // carries the query; the result block carries the encrypted ++ // content Anthropic requires verbatim on later turns. ++ else if (block.type === "hostedSearch" && block.phase === "server_tool_use") { ++ blocks.push({ ++ type: "server_tool_use", ++ id: block.blockId, ++ name: block.name, ++ input: block.input ?? {}, ++ }); ++ } ++ else if (block.type === "hostedSearch" && block.phase === "web_search_tool_result") { ++ const wire = block.wire; ++ if (wire && typeof wire === "object") { ++ const replay = { ...wire }; ++ delete replay.type; ++ blocks.push({ ++ type: "web_search_tool_result", ++ tool_use_id: block.blockId, ++ ...replay, ++ }); ++ } ++ } + } + if (blocks.length === 0) + continue; +diff --git a/dist/api/azure-openai-responses.js b/dist/api/azure-openai-responses.js +index d40f2bba38093d88ad8e6a3c80a521f79d231ed2..1431552a1df48732b972406541f3e2ab3558918f 100644 +--- a/dist/api/azure-openai-responses.js ++++ b/dist/api/azure-openai-responses.js +@@ -223,6 +223,12 @@ function buildParams(model, context, options, deploymentName, grammarToolInputPr + if (options?.toolChoice !== undefined) { + params.tool_choice = options.toolChoice; + } ++ // Hosted web search (PI-Desktop): same opt-in as openai-responses; the ++ // azure adapter reuses the shared stream processor, which already knows ++ // how to extract web_search_call items. ++ if (model.webSearch === true) { ++ params.tools = [...(params.tools ?? []), { type: "web_search" }]; ++ } + if (model.reasoning) { + if (options?.reasoningEffort || options?.reasoningSummary) { + const effort = options?.reasoningEffort +@@ -240,6 +246,12 @@ function buildParams(model, context, options, deploymentName, grammarToolInputPr + }; + } + } ++ // Hosted web search (PI-Desktop): sources ride the opt-in `include` ++ // channel; merged here because the reasoning block above assigns ++ // `params.include` outright. ++ if (model.webSearch === true) { ++ params.include = [...new Set([...(params.include ?? []), "web_search_call.action.sources"])]; ++ } + // Last so custom keys override the named request fields. + if (options?.samplingParams) { + Object.assign(params, options.samplingParams); diff --git a/dist/api/openai-completions.js b/dist/api/openai-completions.js index 48464f2bf56b9369d5c202e4f02fc4f13d35b412..4d7105cd6441115804517156c8c8584e02dde238 100644 --- a/dist/api/openai-completions.js @@ -79,10 +249,94 @@ index 48464f2bf56b9369d5c202e4f02fc4f13d35b412..4d7105cd6441115804517156c8c8584e } function convertTools(tools, compat) { diff --git a/dist/api/openai-responses-shared.js b/dist/api/openai-responses-shared.js -index 43e463dbfd1e6cc437ebc680468036f47678de26..364c8c0426d12543c1b64daa8e6eaa22f35f6f60 100644 +index 43e463dbfd1e6cc437ebc680468036f47678de26..c024c08ace55b7b7f0f9ef5f6c3514dd04f95ca0 100644 --- a/dist/api/openai-responses-shared.js +++ b/dist/api/openai-responses-shared.js -@@ -636,6 +636,10 @@ export async function processResponsesStream(openaiStream, output, stream, model +@@ -162,6 +162,11 @@ export function convertResponsesMessages(model, context, allowedToolCallProvider + phase: parsedSignature?.phase, + }); + } ++ // Hosted web search (PI-Desktop): replay the search item in ++ // place so follow-up turns keep the search state. ++ else if (block.type === "hostedSearch" && block.phase === "web_search_call" && isSameModel) { ++ output.push({ ...(block.wire ?? { type: "web_search_call", id: block.blockId, status: "completed" }) }); ++ } + else if (block.type === "toolCall") { + const toolCall = block; + const [callId, itemIdRaw] = toolCall.id.split("|"); +@@ -405,6 +410,28 @@ export async function processResponsesStream(openaiStream, output, stream, model + stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output }); + return slot; + } ++ // Hosted web search (PI-Desktop): a server-side search call. The raw ++ // item is kept whole; `response.web_search_call.*` status events and ++ // the done item update the same slot. Replay pushes the item back as ++ // an output item so multi-turn search state survives. ++ if (item.type === "web_search_call") { ++ const block = { ++ type: "hostedSearch", ++ phase: "web_search_call", ++ blockId: item.id, ++ status: item.status ?? "in_progress", ++ wire: item, ++ }; ++ output.content.push(block); ++ const slot = { ++ type: "hostedSearch", ++ block, ++ contentIndex: output.content.length - 1, ++ }; ++ outputSlots.set(outputIndex, slot); ++ stream.push({ type: "hosted_search_update", contentIndex: slot.contentIndex, partial: output }); ++ return slot; ++ } + return undefined; + }; + const getOrCreateSlot = (outputIndex, item) => { +@@ -529,6 +556,27 @@ export async function processResponsesStream(openaiStream, output, stream, model + partial: output, + }); + } ++ // Hosted web search (PI-Desktop): citation annotations arrive while ++ // the text streams. They are collected on the output message so the ++ // application can render sources without parsing text offsets. ++ else if (event.type === "response.output_text.annotation.added") { ++ const annotation = event.annotation; ++ if (annotation?.type === "url_citation" && typeof annotation.url === "string") { ++ output.hostedSearchCitations = output.hostedSearchCitations ?? []; ++ output.hostedSearchCitations.push({ ++ url: annotation.url, ++ title: typeof annotation.title === "string" ? annotation.title : undefined, ++ }); ++ } ++ } ++ // Hosted web search status transitions (`response.web_search_call.in_progress` ++ // / `.completed` / `.failed`); the payload mirrors the item. ++ else if (typeof event.type === "string" && event.type.startsWith("response.web_search_call.")) { ++ const item = event.item ?? event.output_item; ++ if (item && typeof item === "object" && item.type === "web_search_call") { ++ getOrCreateSlot(event.output_index, item); ++ } ++ } + else if (event.type === "response.refusal.delta") { + const slot = getSlot(event.output_index, "text"); + if (!slot) +@@ -633,9 +681,25 @@ export async function processResponsesStream(openaiStream, output, stream, model + }); + outputSlots.delete(event.output_index); + } ++ // Hosted web search (PI-Desktop): finalize the slot with the ++ // done item so replay carries the server's final representation. ++ else if (item.type === "web_search_call" && slot?.type === "hostedSearch") { ++ slot.block.status = item.status ?? slot.block.status; ++ slot.block.wire = item; ++ stream.push({ ++ type: "hosted_search_update", ++ contentIndex: slot.contentIndex, ++ partial: output, ++ }); ++ outputSlots.delete(event.output_index); ++ } } else if (event.type === "response.completed" || event.type === "response.incomplete") { finalizeResponse(event.response); @@ -93,6 +347,36 @@ index 43e463dbfd1e6cc437ebc680468036f47678de26..364c8c0426d12543c1b64daa8e6eaa22 } else if (event.type === "error") { throw new Error(`Error Code ${event.code}: ${event.message}` || "Unknown error"); +diff --git a/dist/api/openai-responses.js b/dist/api/openai-responses.js +index 52fcb306b5512c99518e7a05cd99d6ac55c7c8f0..7c612528ae8138cb84bbd20318cef6d7ebce7532 100644 +--- a/dist/api/openai-responses.js ++++ b/dist/api/openai-responses.js +@@ -247,6 +247,12 @@ function buildParams(model, context, options, compat = getCompat(model), grammar + supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools, + }); + } ++ // Hosted web search (PI-Desktop): attach the server-side tool when the ++ // model record opts in. The Responses `web_search` tool definition is a ++ // bare tagged object; it never passes through convertResponsesTools. ++ if (model.webSearch === true) { ++ params.tools = [...(params.tools ?? []), { type: "web_search" }]; ++ } + if (options?.toolChoice !== undefined) { + params.tool_choice = options.toolChoice; + } +@@ -269,6 +275,12 @@ function buildParams(model, context, options, compat = getCompat(model), grammar + if (model.provider === "xai") + params.include = ["reasoning.encrypted_content"]; + } ++ // Hosted web search (PI-Desktop): sources ride the opt-in `include` ++ // channel; merged here because the reasoning block above assigns ++ // `params.include` outright. ++ if (model.webSearch === true) { ++ params.include = [...new Set([...(params.include ?? []), "web_search_call.action.sources"])]; ++ } + // Last so custom keys override the named request fields. + if (options?.samplingParams) { + Object.assign(params, options.samplingParams); diff --git a/dist/auth/oauth/anthropic.js b/dist/auth/oauth/anthropic.js index 8692e906caa2609396bdccae98d4d593e024ecf6..029138715f58dbb2e5825f9a2d0c182ed7e81f2a 100644 --- a/dist/auth/oauth/anthropic.js diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 70b00c7942..b0de56fc00 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,7 +15,8 @@ overrides: vite@5: '>=6.4.3 <7' patchedDependencies: - '@earendil-works/pi-ai@0.85.1': 7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376 + '@earendil-works/pi-agent-core@0.85.1': d6395b14bbbfd75778197b1a6a0d96b1b2c9f39c7c317c2291392d4a7dda183b + '@earendil-works/pi-ai@0.85.1': 3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab importers: @@ -36,7 +37,7 @@ importers: devDependencies: '@earendil-works/pi-ai': specifier: 0.85.1 - version: 0.85.1(patch_hash=7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + version: 0.85.1(patch_hash=3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) '@pi-desktop/agent-host': specifier: workspace:* version: link:../../packages/agent-host @@ -219,10 +220,10 @@ importers: dependencies: '@earendil-works/pi-agent-core': specifier: 0.85.1 - version: 0.85.1(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + version: 0.85.1(patch_hash=d6395b14bbbfd75778197b1a6a0d96b1b2c9f39c7c317c2291392d4a7dda183b)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) '@earendil-works/pi-ai': specifier: 0.85.1 - version: 0.85.1(patch_hash=7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + version: 0.85.1(patch_hash=3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) '@earendil-works/pi-coding-agent': specifier: 0.85.1 version: 0.85.1(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) @@ -4769,10 +4770,10 @@ snapshots: dependencies: esbuild: 0.28.1 - '@earendil-works/pi-agent-core@0.85.1(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.85.1(patch_hash=d6395b14bbbfd75778197b1a6a0d96b1b2c9f39c7c317c2291392d4a7dda183b)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3)': dependencies: '@earendil-works/chord': 0.85.1 - '@earendil-works/pi-ai': 0.85.1(patch_hash=7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.85.1(patch_hash=3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) '@earendil-works/pi-telemetry': 0.85.1 diff: 8.0.4 ignore: 7.0.5 @@ -4786,7 +4787,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.85.1(patch_hash=7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.85.1(patch_hash=3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.123.0(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -4809,8 +4810,8 @@ snapshots: '@earendil-works/pi-coding-agent@0.85.1(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3)': dependencies: '@earendil-works/chord': 0.85.1 - '@earendil-works/pi-agent-core': 0.85.1(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.85.1(patch_hash=7f0a27b6b948cdfcd19c487093def4b4795865c712e5fc1a578ae77636eb4376)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.85.1(patch_hash=d6395b14bbbfd75778197b1a6a0d96b1b2c9f39c7c317c2291392d4a7dda183b)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.85.1(patch_hash=3082768ed4bca473884a4a90cbb263d260c6884fdbc3ec039509da5ae9fcdcab)(supports-color@7.2.0)(ws@8.21.1)(zod@4.4.3) '@earendil-works/pi-tui': 0.85.1 '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 312db6d1f5..e33f8465d4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -44,4 +44,5 @@ minimumReleaseAgeExclude: - '@earendil-works/pi-ai@0.85.1' - '@earendil-works/pi-telemetry@0.85.1' patchedDependencies: + '@earendil-works/pi-agent-core@0.85.1': patches/@earendil-works__pi-agent-core@0.85.1.patch '@earendil-works/pi-ai@0.85.1': patches/@earendil-works__pi-ai@0.85.1.patch From 003bc0aebfa205331371652d6d0e67ecd026766d Mon Sep 17 00:00:00 2001 From: vastsa Date: Sat, 19 Sep 2026 14:57:05 +0800 Subject: [PATCH 2/2] fix(runtime): persist hosted search blocks for restart replay Live turns already kept the adapter's raw search parts in memory. Restart rebuilt history from display rounds only, so Anthropic encrypted_content and Responses web_search_call items vanished and later turns lost grounding. Store those parts on hostedSearch.replay and restore them in historyToEntries. --- .../settings/ModelSelectionPanes.tsx | 6 +- crates/host-core/src/sessions.rs | 27 +++++ ...er-hosted-web-search-adapter-capability.md | 20 ++-- docs/spec/03-runtime/04-data-storage.md | 5 +- .../03-runtime/11-provider-model-system.md | 16 ++- docs/zh-CN/spec/03-runtime/04-data-storage.md | 5 +- .../03-runtime/11-provider-model-system.md | 15 ++- packages/agent-runtime/src/runtime.test.ts | 71 +++++++++++ packages/agent-runtime/src/runtime.ts | 29 +++-- packages/shared/src/model-config-import.ts | 3 + packages/shared/src/native-web-search.test.ts | 110 ++++++++++++++++++ packages/shared/src/native-web-search.ts | 86 +++++++++++--- packages/shared/src/types/messages.ts | 19 +++ 13 files changed, 362 insertions(+), 50 deletions(-) diff --git a/apps/desktop/src/components/settings/ModelSelectionPanes.tsx b/apps/desktop/src/components/settings/ModelSelectionPanes.tsx index 78e88a77df..2c18661c2c 100644 --- a/apps/desktop/src/components/settings/ModelSelectionPanes.tsx +++ b/apps/desktop/src/components/settings/ModelSelectionPanes.tsx @@ -16,6 +16,7 @@ import { bindingFromModelInfo, formatTokenCount, modelMatchesFilter, + nativeWebSearchSupportedOn, publishedThinkingLevels, sortThinkingLevels, type ModelBinding, @@ -252,11 +253,10 @@ export function ModelSelectionPanes({ if (models.length === 0) setChosenQuery(""); }, [models.length]); - // The hosted web search tool only exists on two wire APIs; on any other + // The hosted web search tool only exists on two wires; on any other // style the opt-in cannot work, so the checkbox stays present but disabled // with an explanatory hint instead of silently doing nothing. - const nativeWebSearchWireCapable = - apiStyle === "responses" || apiStyle === "anthropic_messages"; + const nativeWebSearchWireCapable = nativeWebSearchSupportedOn(apiStyle); /** * The chosen list narrows with the discovered list's rule plus the binding's diff --git a/crates/host-core/src/sessions.rs b/crates/host-core/src/sessions.rs index a5b869186b..22f73d3d2e 100644 --- a/crates/host-core/src/sessions.rs +++ b/crates/host-core/src/sessions.rs @@ -4498,6 +4498,15 @@ mod tests { { "url": "https://example.com/a", "title": "A" } ] } + ], + "replay": [ + { + "type": "hostedSearch", + "phase": "server_tool_use", + "blockId": "srvtoolu_01", + "name": "web_search", + "input": { "query": "pi-desktop release notes" } + } ] })), session_message: None, @@ -4521,6 +4530,15 @@ mod tests { { "url": "https://example.com/a", "title": "A" } ] } + ], + "replay": [ + { + "type": "hostedSearch", + "phase": "server_tool_use", + "blockId": "srvtoolu_01", + "name": "web_search", + "input": { "query": "pi-desktop release notes" } + } ] }) ); @@ -4539,6 +4557,15 @@ mod tests { { "url": "https://example.com/a", "title": "A" } ] } + ], + "replay": [ + { + "type": "hostedSearch", + "phase": "server_tool_use", + "blockId": "srvtoolu_01", + "name": "web_search", + "input": { "query": "pi-desktop release notes" } + } ] })) ); diff --git a/docs/adr/0296-provider-hosted-web-search-adapter-capability.md b/docs/adr/0296-provider-hosted-web-search-adapter-capability.md index ba56066d06..15a21fc98f 100644 --- a/docs/adr/0296-provider-hosted-web-search-adapter-capability.md +++ b/docs/adr/0296-provider-hosted-web-search-adapter-capability.md @@ -69,15 +69,16 @@ Two upstream facts forced the design: the pi-ai `Model`, so no new IPC, sidecar parameter, or runtime rebuild hook is needed. -4. **Display data and replay data are separated.** The runtime normalizes - `hostedSearch` blocks into `UiMessage.hostedSearch` - (`status/rounds[]`, one round per provider search/open-page/find-in-page - call, shape-checked, junk ignored) for the activity rows; the raw wire blocks stay inside - pi-ai's message content for replay. Persistence is an additive +4. **Display data and replay data are separated, and both persist.** The + runtime normalizes `hostedSearch` blocks into `UiMessage.hostedSearch` + (`status/rounds[]` for activity rows; `replay[]` for the adapter's raw + content parts, stripped of streaming scratch). Persistence is an additive `hostedSearch` transcript block (`ui_to_record` / `record_to_ui`) — no SQL - migration, per the storage playbook. The wire-slimming functions - (`streamingMessageIdentity`, `applyMessageUpdate`) carry the field so - delta frames cannot drop it. + migration. `historyToEntries` restores `replay` after thinking and before + text so convertMessages can ground later turns after a restart. Transcripts + written without `replay` still render; they just cannot ground. The + wire-slimming functions (`streamingMessageIdentity`, `applyMessageUpdate`) + carry the field so delta frames cannot drop it. 5. **UI is a transcript activity row per round, not a composer control.** Each round of a `hostedSearch` activity renders as a `HostedSearchRow` on @@ -103,3 +104,6 @@ Two upstream facts forced the design: limitation to revisit with the upstream patch. - Compaction rewrites history and drops search blocks; later turns lose old grounding and the model searches again as needed. Documented behavior. +- Search executes on the provider. There is no local fetch, no ask/allow + prompt, and billing is the provider's. The model-level opt-in (default + off) is the user consent surface. diff --git a/docs/spec/03-runtime/04-data-storage.md b/docs/spec/03-runtime/04-data-storage.md index 72081c12f0..2b93813ff4 100644 --- a/docs/spec/03-runtime/04-data-storage.md +++ b/docs/spec/03-runtime/04-data-storage.md @@ -756,7 +756,10 @@ type Block = status: "searching" | "completed" | "failed"; kind?: "search" | "openPage" | "findInPage"; query?: string; url?: string; - sources: Array<{ url: string; title?: string }> }> }; + sources: Array<{ url: string; title?: string }> }>; + replay?: Array<{ type: "hostedSearch"; phase: string; + blockId?: string; name?: string; input?: unknown; + status?: string; isError?: boolean; wire?: unknown }> }; ``` - Tool results are stored **post-truncation** (16-tool-result-limits); full diff --git a/docs/spec/03-runtime/11-provider-model-system.md b/docs/spec/03-runtime/11-provider-model-system.md index e116d52649..cf27106929 100644 --- a/docs/spec/03-runtime/11-provider-model-system.md +++ b/docs/spec/03-runtime/11-provider-model-system.md @@ -226,13 +226,17 @@ PI-Desktop must not permanently restrict users to a short fixed model list. explanatory copy is required. 10a. `nativeWebSearch` is a two-state opt-in (absent means off; there is no catalog baseline because models.dev publishes no hosted-tool capability). - When enabled and the provider's resolved wire API is - `anthropic_messages` or `responses`, the adapter attaches the provider's + When enabled and the model's resolved wire API is `anthropic-messages`, + `openai-responses`, or `azure-openai-responses` (stored apiStyle + `anthropic_messages` / `responses`), the adapter attaches the provider's hosted web search tool (`web_search_20250305` / `web_search`), extracts - the search activity into `UiMessage.hostedSearch`, and replays the raw - search blocks on later turns (ADR 0296). The checkbox is disabled when - the provider's API style is neither of those two. Gateways that do not - support the tool surface the provider error; the remedy is unchecking. + the search activity into `UiMessage.hostedSearch` (`rounds` for display, + `replay` for convertMessages), and restores those raw blocks on later + turns including after a restart (ADR 0296). The checkbox is disabled + when the provider's API style is neither of those two. Gateways that do + not support the tool surface the provider error; the remedy is unchecking. + Search runs on the provider: there is no local fetch and no permission + prompt. Compaction still drops search blocks. 11. `ModelInfo` is the published record the settings surface compares against, so a stored binding must not shape its capabilities or reasoning fields. Effective limits, reasoning and thinking levels are resolved through the diff --git a/docs/zh-CN/spec/03-runtime/04-data-storage.md b/docs/zh-CN/spec/03-runtime/04-data-storage.md index e16c6932b2..7854d8553b 100644 --- a/docs/zh-CN/spec/03-runtime/04-data-storage.md +++ b/docs/zh-CN/spec/03-runtime/04-data-storage.md @@ -680,7 +680,10 @@ type Block = status: "searching" | "completed" | "failed"; kind?: "search" | "openPage" | "findInPage"; query?: string; url?: string; - sources: Array<{ url: string; title?: string }> }> }; + sources: Array<{ url: string; title?: string }> }>; + replay?: Array<{ type: "hostedSearch"; phase: string; + blockId?: string; name?: string; input?: unknown; + status?: string; isError?: boolean; wire?: unknown }> }; ``` - 工具结果存储**截断后**(16 个工具结果限制);满 diff --git a/docs/zh-CN/spec/03-runtime/11-provider-model-system.md b/docs/zh-CN/spec/03-runtime/11-provider-model-system.md index 591d0ab664..78b4948b67 100644 --- a/docs/zh-CN/spec/03-runtime/11-provider-model-system.md +++ b/docs/zh-CN/spec/03-runtime/11-provider-model-system.md @@ -202,12 +202,15 @@ PI-Desktop 不得把用户永久限制在一份简短的固定模型列表上。 值,存下来的是"跟随目录",而不是一个取值相同的覆盖。因此与 models.dev 保持一致本身就是重置,不需要另外的重置控件,也不需要逐项能力的解释文案。 10a. `nativeWebSearch` 是两态的主动开启(缺省即关闭;没有目录基线,因为 - models.dev 不发布托管工具能力)。启用且提供商解析后的线路 API 是 - `anthropic_messages` 或 `responses` 时,适配器会附加提供商托管的联网 - 搜索工具(`web_search_20250305` / `web_search`),把搜索活动提取为 - `UiMessage.hostedSearch`,并在后续回合回放原始搜索块(ADR 0296)。 - 提供商接口风格不属于这两种时复选框禁用。不支持该工具的网关会把 - 提供商错误暴露出来;处理方式是取消勾选。 + models.dev 不发布托管工具能力)。启用且模型解析后的线路 API 是 + `anthropic-messages`、`openai-responses` 或 `azure-openai-responses` + (存储的 apiStyle 为 `anthropic_messages` / `responses`)时,适配器会 + 附加提供商托管的联网搜索工具(`web_search_20250305` / `web_search`), + 把搜索活动提取为 `UiMessage.hostedSearch`(`rounds` 用于展示,`replay` + 用于 convertMessages),并在后续回合——包括重启之后——回放这些原始 + 搜索块(ADR 0296)。提供商接口风格不属于这两种时复选框禁用。不支持 + 该工具的网关会把提供商错误暴露出来;处理方式是取消勾选。搜索在提供商 + 侧执行:没有本地抓取,也没有权限询问。压缩仍会丢掉搜索块。 11. `ModelInfo` 是设置界面用来对照的已发布记录,因此已存储的 binding 不得 塑造它的能力或推理字段。有效上限、推理与思考级别都通过那个确切的 binding 解析;有效的传输模态数组还会额外套用显式的附件覆盖。 diff --git a/packages/agent-runtime/src/runtime.test.ts b/packages/agent-runtime/src/runtime.test.ts index 6ef74f4c73..fae90e337b 100644 --- a/packages/agent-runtime/src/runtime.test.ts +++ b/packages/agent-runtime/src/runtime.test.ts @@ -8288,7 +8288,78 @@ describe("DesktopAgentRuntime hosted web search rounds (ADR 0296)", () => { }, { id: "ws_2", status: "completed", query: "second query", sources: [] }, ], + replay: [ + round("ws_1", "completed", "first query", [ + { url: "https://example.com/a", title: "A" }, + ]), + round("ws_2", "in_progress", "second query"), + ], }); await runtime.dispose(); }); + + it("replays persisted hostedSearch blocks into model context on restore", async () => { + const restored = createRuntime({ + history: [ + { + id: "u1", + role: "user", + content: "news?", + status: "complete", + createdAt: "2026-09-19T00:00:00.000Z", + }, + { + id: "a1", + role: "assistant", + content: "here is the news", + status: "complete", + createdAt: "2026-09-19T00:00:01.000Z", + hostedSearch: { + status: "completed", + rounds: [ + { id: "srvtoolu_01", status: "completed", query: "news", sources: [] }, + ], + replay: [ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvtoolu_01", + name: "web_search", + input: { query: "news" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvtoolu_01", + wire: { + type: "web_search_tool_result", + encrypted_content: "enc-1", + }, + }, + ], + }, + }, + ], + }); + const assistant = (restored as unknown as { agent: Agent }).agent.state.messages.find( + (message) => message.role === "assistant", + ) as { content: unknown[] } | undefined; + expect(assistant?.content).toEqual([ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvtoolu_01", + name: "web_search", + input: { query: "news" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvtoolu_01", + wire: { type: "web_search_tool_result", encrypted_content: "enc-1" }, + }, + { type: "text", text: "here is the news" }, + ]); + await restored.dispose(); + }); }); diff --git a/packages/agent-runtime/src/runtime.ts b/packages/agent-runtime/src/runtime.ts index f2047c3898..572831bf86 100644 --- a/packages/agent-runtime/src/runtime.ts +++ b/packages/agent-runtime/src/runtime.ts @@ -96,7 +96,7 @@ import { DEFAULT_SUBAGENT_PERMISSION, formatAskToolOutput, formatSessionMessage, - hostedSearchFromBlocks, + hostedSearchFromMessage, isCommandShellOption, isToolsOutputParams, MAX_SUBAGENT_CONCURRENCY, @@ -2474,13 +2474,13 @@ Delegation rules: } /* Rebuild pi-ai messages from the persisted transcript, including tool - * call/result pairs — tool rows persist toolCallId/toolName/toolArgs and - * the result (including deferred-tool activation markers), which is - * everything the model context needs. Losing them - * (the pre-D120 behavior) collapsed a reseeded session to bare chat text: - * the model forgot every file it had read and, seeing its own history - * "answer" without visible tool use, stopped calling tools altogether. - * Failed assistant turns stay transcript-only. */ + * call/result pairs and hosted-search replay blocks. Tool rows persist + * toolCallId/toolName/toolArgs and the result (including deferred-tool + * activation markers). Hosted search persists the adapter's raw content + * parts on `hostedSearch.replay` so Anthropic/Responses can ground later + * turns after a restart. Losing either (the pre-D120 tool behavior) + * collapsed a reseeded session to bare chat text. Failed assistant turns + * stay transcript-only. */ private historyToEntries(history: UiMessage[]): MessageEntry[] { const api = apiBindingForProviderModel(this.provider).api; const deepSeekCompletionsReplay = @@ -2550,6 +2550,15 @@ Delegation rules: : {}), }); } + const replay = m.hostedSearch?.replay; + if (Array.isArray(replay)) { + for (const block of replay) { + if (!block || typeof block !== "object" || block.type !== "hostedSearch") { + continue; + } + content.push(block as unknown as AssistantMessage["content"][number]); + } + } if (m.content?.trim()) { content.push({ type: "text" as const, text: m.content }); } @@ -6610,7 +6619,7 @@ Delegation rules: content?: unknown; hostedSearchCitations?: unknown; }; - const next = hostedSearchFromBlocks({ + const next = hostedSearchFromMessage({ content: record?.content, citations: record?.hostedSearchCitations, }); @@ -6815,7 +6824,7 @@ Delegation rules: ); } const usage = usageFromPi((event.message as any).usage as Usage | undefined); - const hostedSearch = hostedSearchFromBlocks({ + const hostedSearch = hostedSearchFromMessage({ content: (event.message as any).content, citations: (event.message as any).hostedSearchCitations, }); diff --git a/packages/shared/src/model-config-import.ts b/packages/shared/src/model-config-import.ts index a6561dcff7..43655a1b97 100644 --- a/packages/shared/src/model-config-import.ts +++ b/packages/shared/src/model-config-import.ts @@ -828,6 +828,9 @@ function bindingFromGenericModel( importedContextWindowSource(record?.contextWindowSource) ?? (contextWindow === undefined ? base.contextWindowSource : "user"), maxTokens: maxTokens ?? base.maxTokens, + ...(record?.nativeWebSearch === true || record?.native_web_search === true + ? { nativeWebSearch: true } + : {}), }; } diff --git a/packages/shared/src/native-web-search.test.ts b/packages/shared/src/native-web-search.test.ts index c918dc6507..a3db453ad1 100644 --- a/packages/shared/src/native-web-search.test.ts +++ b/packages/shared/src/native-web-search.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from "vitest"; import { hostedSearchFromBlocks, + hostedSearchFromMessage, + hostedSearchReplayBlocks, hostedSearchRounds, + nativeWebSearchSupportedOn, nativeWebSearchToolFor, resolveNativeWebSearch, } from "./native-web-search.js"; @@ -463,3 +466,110 @@ describe("hostedSearchRounds", () => { ).toEqual([]); }); }); + +describe("nativeWebSearchSupportedOn", () => { + it("accepts stored apiStyle and resolved wire spellings", () => { + expect(nativeWebSearchSupportedOn("responses")).toBe(true); + expect(nativeWebSearchSupportedOn("anthropic_messages")).toBe(true); + expect(nativeWebSearchSupportedOn("openai-responses")).toBe(true); + expect(nativeWebSearchSupportedOn("azure-openai-responses")).toBe(true); + expect(nativeWebSearchSupportedOn("anthropic-messages")).toBe(true); + }); + + it("rejects wires without a hosted search tool", () => { + expect(nativeWebSearchSupportedOn("chat_completions")).toBe(false); + expect(nativeWebSearchSupportedOn("openai-completions")).toBe(false); + expect(nativeWebSearchSupportedOn("openai_codex_responses")).toBe(false); + expect(nativeWebSearchSupportedOn(undefined)).toBe(false); + expect(nativeWebSearchSupportedOn("")).toBe(false); + }); +}); + +describe("hostedSearchReplayBlocks", () => { + it("keeps wire payloads and drops streaming scratch", () => { + expect( + hostedSearchReplayBlocks([ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvtoolu_01", + name: "web_search", + input: { query: "pi-desktop" }, + index: 2, + inputJson: "{\"query\":\"pi-desktop\"}", + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvtoolu_01", + wire: { + type: "web_search_tool_result", + encrypted_content: "enc-1", + }, + }, + { type: "text", text: "answer" }, + ]), + ).toEqual([ + { + type: "hostedSearch", + phase: "server_tool_use", + blockId: "srvtoolu_01", + name: "web_search", + input: { query: "pi-desktop" }, + }, + { + type: "hostedSearch", + phase: "web_search_tool_result", + blockId: "srvtoolu_01", + wire: { + type: "web_search_tool_result", + encrypted_content: "enc-1", + }, + }, + ]); + }); + + it("ignores junk and empty input", () => { + expect(hostedSearchReplayBlocks(undefined)).toEqual([]); + expect(hostedSearchReplayBlocks([{ type: "hostedSearch" }])).toEqual([]); + expect(hostedSearchReplayBlocks("nope")).toEqual([]); + }); +}); + +describe("hostedSearchFromMessage", () => { + it("attaches replay next to the display rounds", () => { + const content = [ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "q" }, + }, + }, + ]; + const search = hostedSearchFromMessage({ content }); + expect(search?.rounds).toEqual([ + { id: "ws_1", status: "completed", query: "q", sources: [] }, + ]); + expect(search?.replay).toEqual([ + { + type: "hostedSearch", + phase: "web_search_call", + blockId: "ws_1", + status: "completed", + wire: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "q" }, + }, + }, + ]); + }); +}); + diff --git a/packages/shared/src/native-web-search.ts b/packages/shared/src/native-web-search.ts index 53e9c53eb8..a94b34482f 100644 --- a/packages/shared/src/native-web-search.ts +++ b/packages/shared/src/native-web-search.ts @@ -1,20 +1,14 @@ /** - * Single evaluation point for the provider-hosted web search tool. + * Evaluation helpers for the provider-hosted web search tool. * - * Every consumer — the request assembly in the pi-ai adapters, the runtime, - * and any UI gating — must resolve "does this request carry the hosted web - * search tool" through `resolveNativeWebSearch` and nothing else. Detection - * inputs are deliberately minimal: - * - * - `wireApi` is the RESOLVED wire API for the model (the result of - * `apiBindingForProviderModel`), never the provider's stored apiStyle: a - * model-level catalog pin can select a different wire than the provider - * row, and judging by the stored style produced UI/runtime disagreement - * before. - * - `modelWebSearch` is the `ModelConfig.webSearch` flag derived from the - * user's per-model binding opt-in. models.dev publishes no hosted-tool - * capability, so there is no catalog default: only an explicit user - * opt-in can enable the tool. + * - `nativeWebSearchSupportedOn` gates the settings checkbox. It accepts + * stored apiStyle (`responses`, `anthropic_messages`) and resolved wire + * APIs (`openai-responses`, `anthropic-messages`, `azure-openai-responses`). + * - `resolveNativeWebSearch` is the runtime decision: capable wire AND the + * binding opt-in. Adapters then key on `model.webSearch`, which + * `modelConfigWithBinding` copies from that opt-in. + * - `hostedSearchFromMessage` is what the runtime persists: display rounds + * plus raw `replay` blocks for convertMessages after a restart. * * Vendor display names, base URL hostnames, and model id substrings are * intentionally not consulted. An endpoint either carries the tool on the @@ -65,6 +59,68 @@ export function nativeWebSearchToolFor( return undefined; } +const NATIVE_WEB_SEARCH_API_STYLES = new Set(["responses", "anthropic_messages"]); + +/** + * Whether a stored apiStyle or a resolved wire API can carry the hosted + * search tool. Settings UI gates the checkbox with this; it accepts both + * spellings because the pane sees `responses` / `anthropic_messages` while + * the runtime sees `openai-responses` / `anthropic-messages`. + */ +export function nativeWebSearchSupportedOn(api: string | undefined): boolean { + const value = (api ?? "").trim().toLowerCase(); + if (!value) return false; + if (NATIVE_WEB_SEARCH_WIRE_APIS.has(value)) return true; + if (NATIVE_WEB_SEARCH_API_STYLES.has(value)) return true; + return NATIVE_WEB_SEARCH_WIRE_APIS.has(value.replace(/_/g, "-")); +} + +/** + * Capture the adapter's hostedSearch content parts for convertMessages + * replay. Streaming scratch (`index`, `inputJson`) is dropped; unknown + * shapes are ignored. Display normalization is `hostedSearchFromBlocks`. + */ +export function hostedSearchReplayBlocks( + content: unknown, +): import("./types/messages.js").HostedSearchReplayBlock[] { + if (!Array.isArray(content)) return []; + const replay: import("./types/messages.js").HostedSearchReplayBlock[] = []; + for (const part of content) { + if (!part || typeof part !== "object") continue; + const block = part as Record; + if (block.type !== "hostedSearch") continue; + const phase = typeof block.phase === "string" ? block.phase.trim() : ""; + if (!phase) continue; + const next: import("./types/messages.js").HostedSearchReplayBlock = { + type: "hostedSearch", + phase, + }; + if (typeof block.blockId === "string" && block.blockId) next.blockId = block.blockId; + if (typeof block.name === "string" && block.name) next.name = block.name; + if (block.input !== undefined) next.input = block.input; + if (typeof block.status === "string") next.status = block.status; + if (block.isError === true) next.isError = true; + if (block.wire && typeof block.wire === "object") next.wire = block.wire; + replay.push(next); + } + return replay; +} + +/** + * Display rounds plus replay payload. Runtime persistence and stream + * updates go through this so a restart can rebuild the same content + * parts convertMessages expects. + */ +export function hostedSearchFromMessage(input: { + content: unknown; + citations?: unknown; +}): import("./types/messages.js").HostedSearch | undefined { + const search = hostedSearchFromBlocks(input); + if (!search) return undefined; + const replay = hostedSearchReplayBlocks(input.content); + return replay.length > 0 ? { ...search, replay } : search; +} + /** * Normalize pi-ai hostedSearch content blocks and message citations into the * shared `HostedSearch` shape: one round per provider search call, in block diff --git a/packages/shared/src/types/messages.ts b/packages/shared/src/types/messages.ts index 3ce6391782..2ba3d124fe 100644 --- a/packages/shared/src/types/messages.ts +++ b/packages/shared/src/types/messages.ts @@ -141,6 +141,25 @@ export type HostedSearch = { * any round is still in flight. */ status: "searching" | "completed" | "failed"; rounds: HostedSearchRound[]; + /** + * Raw pi-ai hostedSearch content parts, in original block order. Display + * uses `rounds`; convertMessages replay after a restart uses this. Absent + * on transcripts written before the field existed — those rows still + * render, but later turns cannot ground on the old search. + */ + replay?: HostedSearchReplayBlock[]; +}; + +/** One adapter-captured search block, stripped of streaming scratch. */ +export type HostedSearchReplayBlock = { + type: "hostedSearch"; + phase: string; + blockId?: string; + name?: string; + input?: unknown; + status?: string; + isError?: boolean; + wire?: unknown; }; /** One provider search round, in the order the provider issued it. */