From 4072b4e212b21dc85004c22a6f3397e6af6ed7f7 Mon Sep 17 00:00:00 2001 From: ahmad-ajmal Date: Wed, 30 Sep 2026 11:05:10 +0100 Subject: [PATCH 01/55] fix: decode and stop silently truncating inbound integration messages --- app/agent_base.py | 22 +- craftos_integrations/README.md | 11 +- craftos_integrations/base.py | 5 + craftos_integrations/helpers/__init__.py | 4 + craftos_integrations/helpers/text.py | 30 ++ craftos_integrations/providers/_shared.py | 1 + .../providers/github/client.py | 11 +- .../providers/gmail/client.py | 126 ++++--- .../providers/gmail/operations.py | 5 +- craftos_integrations/providers/jira/client.py | 11 +- .../providers/outlook/client.py | 24 +- .../providers/slack/client.py | 36 +- .../providers/slack/formatting.py | 70 ++++ .../providers/twitter/client.py | 5 +- .../test_listener_text_fidelity.py | 340 ++++++++++++++++++ tests/integrations/test_provider_listeners.py | 11 +- 16 files changed, 633 insertions(+), 79 deletions(-) create mode 100644 craftos_integrations/helpers/text.py create mode 100644 craftos_integrations/providers/slack/formatting.py create mode 100644 tests/integrations/test_listener_text_fidelity.py diff --git a/app/agent_base.py b/app/agent_base.py index 96c2aea4..43ab346d 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -2446,6 +2446,16 @@ async def _handle_external_event(self, payload: Dict) -> None: integration_type = payload.get("integrationType", "").lower() is_self_message = payload.get("is_self_message", False) + message_id = payload.get("messageId", "") + + # The listener capped a long body (PlatformMessage.truncated): + # mark the cut where both the agent and the chat details see it. + # The agent additionally gets the facts to fetch the rest itself + # (docs/plans/inbound-message-fidelity-plan.md). + truncated = bool(payload.get("truncated")) and bool(message_body) + if truncated: + message_body = f"{message_body}…" + # Normalized attachments (PlatformMessage.attachments) become # descriptor lines with retrieval hints — appended to the body, # or standing in for it on media-only messages so they are no @@ -2518,11 +2528,18 @@ async def _handle_external_event(self, payload: Dict) -> None: location_parts.append(f"channel {channel_id}") location_str = f" in {' / '.join(location_parts)}" if location_parts else "" + # Facts only: the agent decides whether it needs the full body. + truncation_note = "" + if truncated: + id_part = f" Message ID: {message_id}" if message_id else "" + truncation_note = f"(Body truncated.{id_part})\n" + if is_self_message: # Self-message = user is directly talking to the agent via their own platform. event_content = ( f"[USER SELF-MESSAGE via {source}]\n" - f"{message_body}\n\n" + f"{message_body}\n" + f"{truncation_note}\n" f"INSTRUCTIONS: Reply to the message to the user on {source}" f"{account_note}" ) @@ -2538,7 +2555,8 @@ async def _handle_external_event(self, payload: Dict) -> None: f"From: {contact_name} ({contact_id}){location_str}\n" f"Platform: {source}\n" f"{received_on}" - f'Message: "{message_body}"\n\n' + f'Message: "{message_body}"\n' + f"{truncation_note}\n" f"INSTRUCTIONS: Notify the user about this message on their " f"preferred platform (check USER.md 'Preferred Messaging " f"Platform'). If USER.md does not name one, notify via " diff --git a/craftos_integrations/README.md b/craftos_integrations/README.md index 5db195cc..d3cbc3e5 100644 --- a/craftos_integrations/README.md +++ b/craftos_integrations/README.md @@ -169,9 +169,18 @@ has `listen` enabled, and reconciles whenever accounts change. "messageId": "", "is_self_message": False, "raw": {...}, # full original platform event + "attachments": [...], # normalized non-text payloads ({kind, id, name, mime, size, url, extra}) + "truncated": False, # True when the listener capped messageBody } ``` +`messageBody` is plain, decoded text built from the real message body — +listeners unescape HTML entities and render platform markup before +emitting, and never forward an API preview field (Gmail `snippet`, +Outlook `bodyPreview`) as the body. A listener that caps the length uses +`helpers.clip` and sets `truncated`; the host then marks the cut and tells +the agent the message id (`docs/plans/inbound-message-fidelity-plan.md`). + --- ## Configuration: OAuth env vars @@ -399,7 +408,7 @@ For a production-level integration, produce in this order: | 2 | Implement `verify_token` (token auth) or `oauth_spec` (OAuth), plus `identity_of` | `provider.py` | | 3 | Optional: `config_class` + `config_fields` for post-connect knobs | `provider.py` | | 4 | Build the client — one method per endpoint, using `helpers.arequest`, returning `Result` | client in `__init__.py` | -| 5 | Optional: `start_listening` / `stop_listening` (webhook / polling / WebSocket) | client | +| 5 | Optional: `start_listening` / `stop_listening` (webhook / polling / WebSocket). Emit **plain, decoded** text from the real body (unescape entities, render platform markup like `<@U…>`; no API preview fields); cap long bodies with `helpers.clip` and pass its flag as `PlatformMessage.truncated` | client | | 6 | Write `INTEGRATION.md` — identifier shape, silent-drop config flags, auth gotchas | integration root | | 7 | Mirror each client method as a `client_op` with sub-set + umbrella tags | `operations.py` | | 8 | Verify — `scripts/verify_integration.py `, then a live smoke test | see "Verification" | diff --git a/craftos_integrations/base.py b/craftos_integrations/base.py index c6009859..0a9de71c 100644 --- a/craftos_integrations/base.py +++ b/craftos_integrations/base.py @@ -43,6 +43,11 @@ class PlatformMessage: # The HOST formats these into descriptor text + retrieval hints — # listeners only normalize (docs/plans/attachment-reception-plan.md). attachments: List[Dict[str, Any]] = field(default_factory=list) + # True when the listener capped ``text`` (``helpers.clip``). The HOST + # marks the cut and gives the agent the message id to fetch the rest; + # listeners only declare the fact + # (docs/plans/inbound-message-fidelity-plan.md). + truncated: bool = False MessageCallback = Callable[[PlatformMessage], Awaitable[None]] diff --git a/craftos_integrations/helpers/__init__.py b/craftos_integrations/helpers/__init__.py index d004bbd4..36e51584 100644 --- a/craftos_integrations/helpers/__init__.py +++ b/craftos_integrations/helpers/__init__.py @@ -10,15 +10,19 @@ ``status_code → {ok, result} | {error, details}`` envelope shape. result: ``Result`` / ``Ok`` / ``Err`` TypedDict aliases for the envelope — use as return annotations for static type-checking benefits. + text: ``clip`` — word-boundary length cap for listener message bodies + that reports whether it cut (→ ``PlatformMessage.truncated``). """ from .http import arequest, request from .result import Err, Ok, Result +from .text import clip __all__ = [ "Err", "Ok", "Result", "arequest", + "clip", "request", ] diff --git a/craftos_integrations/helpers/text.py b/craftos_integrations/helpers/text.py new file mode 100644 index 00000000..abb8e7c4 --- /dev/null +++ b/craftos_integrations/helpers/text.py @@ -0,0 +1,30 @@ +"""Text shaping for listener message bodies. + +Listeners that cap a body's length (to keep inbound token cost bounded) +use ``clip`` so every cap cuts the same way and reports that it cut — +the flag goes to ``PlatformMessage.truncated`` and the host marks the cut +(docs/plans/inbound-message-fidelity-plan.md). +""" + +from __future__ import annotations + +from typing import Tuple + +# How far back from the limit a word boundary may be before we give up +# and hard-cut (fraction of the limit). +_BOUNDARY_WINDOW = 0.2 + + +def clip(text: str, limit: int) -> Tuple[str, bool]: + """Cut ``text`` to at most ``limit`` chars, preferring a word boundary. + + Returns ``(clipped, was_clipped)``. No ellipsis is added — the host + owns presentation of the cut. + """ + if len(text) <= limit: + return text, False + head = text[:limit] + cut = max(head.rfind(" "), head.rfind("\n")) + if cut >= limit * (1 - _BOUNDARY_WINDOW): + head = head[:cut] + return head.rstrip(), True diff --git a/craftos_integrations/providers/_shared.py b/craftos_integrations/providers/_shared.py index 88bb51cf..a7a06c6f 100644 --- a/craftos_integrations/providers/_shared.py +++ b/craftos_integrations/providers/_shared.py @@ -45,6 +45,7 @@ def platform_message_payload(msg: Any) -> Dict[str, Any]: "is_self_message": raw.get("is_self_message", False), "raw": raw, "attachments": list(getattr(msg, "attachments", None) or []), + "truncated": bool(getattr(msg, "truncated", False)), } diff --git a/craftos_integrations/providers/github/client.py b/craftos_integrations/providers/github/client.py index c1a8f3ca..f96c819f 100644 --- a/craftos_integrations/providers/github/client.py +++ b/craftos_integrations/providers/github/client.py @@ -31,7 +31,7 @@ register_client, save_credential, ) -from ...helpers import Result, arequest +from ...helpers import Result, arequest, clip from ...logger import get_logger logger = get_logger(__name__) @@ -40,6 +40,10 @@ POLL_INTERVAL = 15 RETRY_DELAY = 30 +# Inbound notification comments are previewed at this length (token cost); +# a longer comment is flagged as PlatformMessage.truncated. +_COMMENT_PREVIEW_CHARS = 300 + @dataclass class GitHubCredential: @@ -336,8 +340,10 @@ async def _dispatch_notification(self, notif: Dict[str, Any]) -> None: f"[{repo_full}] {subject_type}: {subject_title}", f"Reason: {reason}", ] + truncated = False if comment_body: - text_parts.append(f"Comment by @{comment_author}: {comment_body[:300]}") + preview, truncated = clip(comment_body, _COMMENT_PREVIEW_CHARS) + text_parts.append(f"Comment by @{comment_author}: {preview}") await self._message_callback( PlatformMessage( @@ -350,6 +356,7 @@ async def _dispatch_notification(self, notif: Dict[str, Any]) -> None: message_id=notif.get("id", ""), timestamp=datetime.now(timezone.utc), raw=notif, + truncated=truncated, ) ) diff --git a/craftos_integrations/providers/gmail/client.py b/craftos_integrations/providers/gmail/client.py index 2384c380..30725b77 100644 --- a/craftos_integrations/providers/gmail/client.py +++ b/craftos_integrations/providers/gmail/client.py @@ -17,6 +17,7 @@ import asyncio import base64 +import html import mimetypes import os import re @@ -27,7 +28,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from html.parser import HTMLParser -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from ... import ( BasePlatformClient, @@ -36,7 +37,7 @@ load_config, register_client, ) -from ...helpers import Result, arequest, request as http_request +from ...helpers import Result, arequest, clip, request as http_request from ...logger import get_logger from .._google_common import ( GoogleApiClientMixin, @@ -49,6 +50,11 @@ POLL_INTERVAL = 5 RETRY_DELAY = 10 +# Incoming mail is forwarded to the agent with its body capped at this many +# chars (token cost); a longer body is flagged as PlatformMessage.truncated +# and the agent can read the rest with get_gmail. +_INBOUND_BODY_CHARS = 2000 + # Headers worth showing the agent. Everything else Gmail returns under # format=full is transport/anti-spam machinery (Received chains, ARC-*, DKIM # signatures, Authentication-Results, SPF, List-Unsubscribe JWTs, Feedback-ID, @@ -190,6 +196,12 @@ def _clean_spaces(text: str) -> str: return _SPACE_RUN.sub(" ", text).strip() +def clean_snippet(raw: str) -> str: + """Gmail's ``snippet`` arrives HTML-escaped (``'`` ``&``); + decode it and collapse filler whitespace.""" + return _clean_spaces(html.unescape(raw or "")) + + def _html_to_text(html: str) -> str: parser = _HtmlText() try: @@ -201,6 +213,51 @@ def _html_to_text(html: str) -> str: return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() +def _walk_payload( + payload: Dict[str, Any], +) -> Tuple[Dict[str, str], List[Dict[str, Any]]]: + """One pass over a ``format=full`` payload: the first decoded + text/plain + text/html bodies (by mime), and every attachmentId part.""" + texts: Dict[str, str] = {} + attachments: List[Dict[str, Any]] = [] + + def _visit(part: Dict[str, Any]) -> None: + body = part.get("body", {}) or {} + mime = part.get("mimeType", "") + if body.get("attachmentId"): + attachments.append( + { + "filename": part.get("filename", ""), + "attachment_id": body["attachmentId"], + "mimeType": mime, + "size": body.get("size", 0), + } + ) + elif ( + mime in ("text/plain", "text/html") + and "data" in body + and not part.get("filename") # inline-data text attachment + and mime not in texts + ): + texts[mime] = _decode_part(part) + for nested in part.get("parts", []) or []: + _visit(nested) + + _visit(payload) + return texts, attachments + + +def _readable_body(texts: Dict[str, str]) -> Tuple[str, str]: + """``(body, body_format)`` from ``_walk_payload`` texts. Prefers the + plain-text alternative; HTML-only mail (most newsletters/notifications) + is converted rather than coming back empty.""" + if texts.get("text/plain", "").strip(): + return texts["text/plain"], "text" + if texts.get("text/html"): + return _html_to_text(texts["text/html"]), "html_converted" + return "", "none" + + GMAIL = IntegrationSpec( name="gmail", cred_class=GoogleCredential, @@ -368,13 +425,15 @@ async def _fetch_and_dispatch(self, msg_id: str) -> None: if not cfg.process_incoming: return - # format=full + a fields partial-response mask: returns headers, - # snippet, and ONLY the parts skeleton (filename/mimeType/ - # attachmentId/size — no body data), staying ~1-3KB. Quota cost is + # format=full + a fields partial-response mask: headers plus the + # parts skeleton with inline body data (the text/plain + text/html + # bodies we read; part headers carry their charset). Attachment bytes never come inline — those parts + # carry only an attachmentId — so the response stays the size of + # the text. The body, not `snippet`, is what we forward: the snippet + # is an HTML-escaped ~200-char preview (issue #444). Quota cost is # flat regardless of format. Three explicit nesting levels cover - # mixed / mixed-inside-signed / one spare; a bare `payload/parts` - # selector would pull body.data too — keep the sub-selection. - _part_sel = "partId,mimeType,filename,body(attachmentId,size)" + # mixed / mixed-inside-signed / one spare. + _part_sel = "partId,mimeType,filename,headers,body(attachmentId,size,data)" result = await arequest( "GET", f"{GMAIL_API_BASE}/users/me/messages/{msg_id}", @@ -383,7 +442,8 @@ async def _fetch_and_dispatch(self, msg_id: str) -> None: ("format", "full"), ( "fields", - "id,threadId,snippet,labelIds,historyId,payload(mimeType,headers," + "id,threadId,labelIds,historyId,payload(mimeType,headers," + "body(size,data)," f"parts({_part_sel},parts({_part_sel},parts({_part_sel}))))", ), ], @@ -398,7 +458,8 @@ async def _fetch_and_dispatch(self, msg_id: str) -> None: } from_header = headers.get("From", "") subject = headers.get("Subject", "(no subject)") - snippet = msg.get("snippet", "") + texts, _ = _walk_payload(msg.get("payload", {}) or {}) + body, truncated = clip(_readable_body(texts)[0].strip(), _INBOUND_BODY_CHARS) sender_name = from_header sender_email = from_header @@ -421,7 +482,7 @@ async def _fetch_and_dispatch(self, msg_id: str) -> None: except Exception: pass - text = f"Subject: {subject}\n{snippet}" if snippet else f"Subject: {subject}" + text = f"Subject: {subject}\n{body}" if body else f"Subject: {subject}" # Real attachments carry a non-empty filename + attachmentId # (Gmail's own paperclip heuristic); nameless attachmentId parts @@ -459,6 +520,7 @@ def _collect(parts: Any) -> None: timestamp=timestamp, raw=msg, attachments=attachments, + truncated=truncated, ) ) @@ -551,7 +613,7 @@ def _shape(msg): # always present and not set by the sender. "internalDate": msg.get("internalDate"), "sizeEstimate": msg.get("sizeEstimate"), - "snippet": _clean_spaces(msg.get("snippet", "")), + "snippet": clean_snippet(msg.get("snippet", "")), # The `metadataHeaders` request param is honoured ONLY for # format=metadata — with format=full (full_body=True) Gmail returns # the entire raw MIME header block. That is ~55% of the payload and @@ -562,44 +624,8 @@ def _shape(msg): "headers": _filter_headers(msg.get("payload", {}).get("headers", [])), } if full_body: - attachments: List[Dict[str, Any]] = [] - texts: Dict[str, str] = {} - - def _visit(part): - body = part.get("body", {}) or {} - mime = part.get("mimeType", "") - if body.get("attachmentId"): - attachments.append( - { - "filename": part.get("filename", ""), - "attachment_id": body["attachmentId"], - "mimeType": mime, - "size": body.get("size", 0), - } - ) - elif ( - mime in ("text/plain", "text/html") - and "data" in body - and not part.get("filename") # inline-data text attachment - and mime not in texts - ): - texts[mime] = _decode_part(part) - for nested in part.get("parts", []) or []: - _visit(nested) - - _visit(msg.get("payload", {}) or {}) - - # Prefer the plain-text alternative; HTML-only mail (most - # newsletters/notifications) used to come back with no body. - if texts.get("text/plain", "").strip(): - email_info["body"] = texts["text/plain"] - email_info["body_format"] = "text" - elif texts.get("text/html"): - email_info["body"] = _html_to_text(texts["text/html"]) - email_info["body_format"] = "html_converted" - else: - email_info["body"] = "" - email_info["body_format"] = "none" + texts, attachments = _walk_payload(msg.get("payload", {}) or {}) + email_info["body"], email_info["body_format"] = _readable_body(texts) email_info["attachments"] = attachments return email_info diff --git a/craftos_integrations/providers/gmail/operations.py b/craftos_integrations/providers/gmail/operations.py index 64aa960f..5777df02 100644 --- a/craftos_integrations/providers/gmail/operations.py +++ b/craftos_integrations/providers/gmail/operations.py @@ -19,6 +19,7 @@ from ...contracts import Operation from .._shared import client_op +from .client import clean_snippet def _get_gmail_thread_op() -> Operation: @@ -80,7 +81,7 @@ async def fn(client: Any, input_data: Dict[str, Any]) -> Dict[str, Any]: "internalDate": msg.get("internalDate"), "labelIds": labels, "unread": "UNREAD" in labels, - "snippet": msg.get("snippet", ""), + "snippet": clean_snippet(msg.get("snippet", "")), } ) res = { @@ -141,7 +142,7 @@ async def fn(client: Any, input_data: Dict[str, Any]) -> Dict[str, Any]: "message_id": msg.get("id"), "to": headers.get("To", ""), "subject": headers.get("Subject", ""), - "snippet": msg.get("snippet", ""), + "snippet": clean_snippet(msg.get("snippet", "")), }, } return res diff --git a/craftos_integrations/providers/jira/client.py b/craftos_integrations/providers/jira/client.py index a474ca85..30decbc1 100644 --- a/craftos_integrations/providers/jira/client.py +++ b/craftos_integrations/providers/jira/client.py @@ -21,7 +21,7 @@ save_config, register_client, ) -from ...helpers import Result, arequest +from ...helpers import Result, arequest, clip from ...logger import get_logger logger = get_logger(__name__) @@ -30,6 +30,10 @@ POLL_INTERVAL = 10 RETRY_DELAY = 15 +# The latest comment on an inbound issue update is previewed at this length +# (token cost); a longer comment is flagged as PlatformMessage.truncated. +_COMMENT_PREVIEW_CHARS = 200 + # Fields requested from the API when the caller wants a lean issue payload # (include_metadata=False and no explicit fields list). Restricting server-side # avoids the full customfield_* dump Jira returns by default. @@ -429,12 +433,14 @@ async def _dispatch_issue(self, issue: Dict[str, Any]) -> None: ] if labels: text_parts.append(f"Labels: {', '.join(labels)}") + truncated = False if comments: latest = comments[-1] cb = _extract_adf_text(latest.get("body", {})) ca = (latest.get("author") or {}).get("displayName", "") if cb: - text_parts.append(f"Latest comment by {ca}: {cb[:200]}") + preview, truncated = clip(cb, _COMMENT_PREVIEW_CHARS) + text_parts.append(f"Latest comment by {ca}: {preview}") timestamp = None try: @@ -456,6 +462,7 @@ async def _dispatch_issue(self, issue: Dict[str, Any]) -> None: timestamp=timestamp, raw=issue, attachments=attachments, + truncated=truncated, ) ) diff --git a/craftos_integrations/providers/outlook/client.py b/craftos_integrations/providers/outlook/client.py index aafbb4aa..6224e482 100644 --- a/craftos_integrations/providers/outlook/client.py +++ b/craftos_integrations/providers/outlook/client.py @@ -19,7 +19,7 @@ register_client, save_credential, ) -from ...helpers import Result, arequest, request as http_request +from ...helpers import Result, arequest, clip, request as http_request from ...logger import get_logger logger = get_logger(__name__) @@ -31,6 +31,11 @@ POLL_INTERVAL = 5 RETRY_DELAY = 10 +# Incoming mail is forwarded to the agent with its body capped at this many +# chars (token cost); a longer body is flagged as PlatformMessage.truncated +# and the agent can read the rest with get_outlook_email. +_INBOUND_BODY_CHARS = 2000 + @dataclass class OutlookCredential: @@ -203,15 +208,20 @@ async def _poll_loop(self) -> None: async def _check_new_messages(self) -> None: if not self._last_poll_time: return + # The full body as plain text, not `bodyPreview` (a 255-char + # preview that reads as the whole message — issue #444). result = await arequest( "GET", f"{GRAPH_API_BASE}/me/messages", - headers=self._auth_header(), + headers={ + **self._auth_header(), + "Prefer": 'outlook.body-content-type="text"', + }, params={ "$filter": f"receivedDateTime ge {self._last_poll_time}", "$orderby": "receivedDateTime asc", "$top": "50", - "$select": "id,from,subject,bodyPreview,receivedDateTime,conversationId,hasAttachments", + "$select": "id,from,subject,body,receivedDateTime,conversationId,hasAttachments", }, expected=(200,), ) @@ -248,8 +258,11 @@ async def _dispatch_message(self, msg: Dict[str, Any]) -> None: return subject = msg.get("subject", "(no subject)") - snippet = msg.get("bodyPreview", "") - text = f"Subject: {subject}\n{snippet}" if snippet else f"Subject: {subject}" + body, truncated = clip( + ((msg.get("body") or {}).get("content") or "").strip(), + _INBOUND_BODY_CHARS, + ) + text = f"Subject: {subject}\n{body}" if body else f"Subject: {subject}" timestamp = None try: @@ -294,6 +307,7 @@ async def _dispatch_message(self, msg: Dict[str, Any]) -> None: timestamp=timestamp, raw=msg, attachments=attachments, + truncated=truncated, ) ) diff --git a/craftos_integrations/providers/slack/client.py b/craftos_integrations/providers/slack/client.py index 3f8c587c..d132c5dc 100644 --- a/craftos_integrations/providers/slack/client.py +++ b/craftos_integrations/providers/slack/client.py @@ -20,6 +20,7 @@ ) from ...helpers import arequest, request as http_request from ...logger import get_logger +from .formatting import to_plain_text logger = get_logger(__name__) @@ -121,6 +122,8 @@ def __init__(self): self._bot_user_id: Optional[str] = None self._last_timestamps: Dict[str, str] = {} self._catchup_done: bool = False + # user id → display name; one entry per workspace member seen. + self._user_names: Dict[str, str] = {} def has_credentials(self) -> bool: return has_credential(self.spec.cred_file) @@ -307,16 +310,8 @@ async def _process_message(self, msg: Dict[str, Any], channel_id: str) -> None: if (not text and not attachments) or user_id == self._bot_user_id: return - sender_name = user_id - try: - info = self.get_user_info(user_id) - if info.get("ok"): - profile = info.get("user", {}).get("profile", {}) - sender_name = ( - profile.get("display_name") or profile.get("real_name") or user_id - ) - except Exception: - pass + sender_name = self._display_name(user_id) or user_id + text = to_plain_text(text, self._display_name) ts_float = float(msg.get("ts", "0")) timestamp = ( @@ -338,6 +333,27 @@ async def _process_message(self, msg: Dict[str, Any], channel_id: str) -> None: ) ) + def _display_name(self, user_id: str) -> str: + """User id → display name, "" when unknown. Memoized per client: + the sender lookup and every ``<@U…>`` mention share one cache. + Failures are not cached, so a transient API error retries.""" + if not user_id: + return "" + cached = self._user_names.get(user_id) + if cached: + return cached + name = "" + try: + info = self.get_user_info(user_id) + if info.get("ok"): + profile = info.get("user", {}).get("profile", {}) + name = profile.get("display_name") or profile.get("real_name") or "" + except Exception: + pass + if name: + self._user_names[user_id] = name + return name + # ----- API ----- async def send_message(self, recipient: str, text: str, **kwargs) -> Dict[str, Any]: payload: Dict[str, Any] = {"channel": recipient, "text": text} diff --git a/craftos_integrations/providers/slack/formatting.py b/craftos_integrations/providers/slack/formatting.py new file mode 100644 index 00000000..fd008ad0 --- /dev/null +++ b/craftos_integrations/providers/slack/formatting.py @@ -0,0 +1,70 @@ +"""Slack message text → readable plain text. + +Slack's ``text`` field is not what the user typed: ``& < >`` arrive +HTML-escaped and every reference is wrapped in angle-bracket markup. +Forwarded as-is, the agent and the chat details show ``<@U0123ABC>`` and +``&`` (issue #444; docs/plans/inbound-message-fidelity-plan.md). + +Pure module — the only outside knowledge (user id → display name) is +injected as ``resolve_user``, so it is testable without a client. +""" + +from __future__ import annotations + +import html +import re +from typing import Callable + +# One `<…>` reference. Slack escapes literal `<` / `>` in user text, so a +# raw angle bracket is always markup. +_REFERENCE = re.compile(r"<([^<>]+)>") + +# / / — label-less special mentions. +_SPECIAL_MENTIONS = {"here", "channel", "everyone"} + + +def to_plain_text(text: str, resolve_user: Callable[[str], str]) -> str: + """Convert Slack references + escapes to plain text. Never raises. + + ============================ ========================================= + ``<@U123>`` / ``<@U123|bob>`` ``@`` (via ``resolve_user``; + falls back to the label, then ``@U123``) + ``<#C123|general>`` ``#general`` (``<#C123>`` → ``#C123``) + ```` ``label (https://x)`` + ```` ``https://x`` (``mailto:`` prefix dropped) + ```` etc. ``@here`` / ``@channel`` / ``@everyone`` + ```` ``@team`` + ```` ``fallback`` + ============================ ========================================= + + References are converted BEFORE unescaping: unescaping first would + turn a user's literal ``<@U1>`` into a fake mention. + """ + if not text: + return "" + return html.unescape( + _REFERENCE.sub(lambda m: _render(m.group(1), resolve_user), text) + ) + + +def _render(ref: str, resolve_user: Callable[[str], str]) -> str: + target, _, label = ref.partition("|") + if target.startswith("@"): + user_id = target[1:] + try: + name = resolve_user(user_id) + except Exception: + name = "" + return f"@{name or label or user_id}" + if target.startswith("#"): + return f"#{label or target[1:]}" + if target.startswith("!"): + keyword = target[1:].split("^", 1)[0] + if keyword in _SPECIAL_MENTIONS: + return f"@{keyword}" + return label or target[1:] + if target.startswith("mailto:"): + target = target[len("mailto:") :] + if label and label != target: + return f"{label} ({target})" + return target diff --git a/craftos_integrations/providers/twitter/client.py b/craftos_integrations/providers/twitter/client.py index ab6a27a6..592025cf 100644 --- a/craftos_integrations/providers/twitter/client.py +++ b/craftos_integrations/providers/twitter/client.py @@ -7,6 +7,7 @@ import base64 import hashlib import hmac +import html import secrets as _secrets import time import urllib.parse @@ -335,7 +336,9 @@ async def _dispatch_mention( att["name"] = media["alt_text"] attachments.append(att) - text = tweet.get("text", "") + # v2 tweet text arrives with & < > HTML-escaped. Decode before the + # watch_tag match too — tags are typed as plain text. + text = html.unescape(tweet.get("text", "")) author_id = tweet.get("author_id", "") author_info = users_map.get(author_id, {}) author_username = author_info.get("username", "") diff --git a/tests/integrations/test_listener_text_fidelity.py b/tests/integrations/test_listener_text_fidelity.py new file mode 100644 index 00000000..d627d1ba --- /dev/null +++ b/tests/integrations/test_listener_text_fidelity.py @@ -0,0 +1,340 @@ +"""Inbound message fidelity (issue #444). + +Listeners emit plain, decoded text built from the real message body, cap +it with ``clip`` and declare the cut as ``PlatformMessage.truncated``; the +host marks the cut for the chat and hands the agent the facts to fetch the +rest (docs/plans/inbound-message-fidelity-plan.md). + +No pytest-asyncio in this repo — async paths are driven with asyncio.run. +""" + +from __future__ import annotations + +import asyncio +import base64 +import time +from types import SimpleNamespace + +import pytest + +import craftos_integrations.providers.gmail.client as gmail_mod +import craftos_integrations.providers.outlook.client as outlook_mod +from craftos_integrations.base import PlatformMessage +from craftos_integrations.helpers import clip +from craftos_integrations.providers._shared import platform_message_payload +from craftos_integrations.providers.gmail.client import clean_snippet +from craftos_integrations.providers.gmail.provider import GmailProvider +from craftos_integrations.providers.outlook.provider import OutlookProvider +from craftos_integrations.providers.slack.formatting import to_plain_text + + +def _b64(text: str) -> str: + return base64.urlsafe_b64encode(text.encode()).decode() + + +def _collect_callback(): + received = [] + + async def callback(msg): + received.append(msg) + + return received, callback + + +# ── clip ──────────────────────────────────────────────────────────────── + + +def test_clip_leaves_short_text_alone(): + assert clip("short", 10) == ("short", False) + assert clip("exactly10!", 10) == ("exactly10!", False) + + +def test_clip_cuts_on_word_boundary(): + text, cut = clip("the quick brown fox jumps", 18) + assert cut is True + assert text == "the quick brown" + + +def test_clip_hard_cuts_without_nearby_boundary(): + text, cut = clip("a " + "x" * 50, 20) + assert cut is True + assert text == ("a " + "x" * 50)[:20] + + +# ── Gmail ─────────────────────────────────────────────────────────────── + + +def test_gmail_clean_snippet_decodes_entities(): + assert clean_snippet("that's Tom & Jerry") == "that's Tom & Jerry" + + +def _gmail_client(monkeypatch, payload): + message = {"id": "m1", "threadId": "t1", "payload": payload} + + async def fake_arequest(method, url, **kwargs): + assert "/users/me/messages/m1" in url + return {"result": message} + + monkeypatch.setattr(gmail_mod, "arequest", fake_arequest) + monkeypatch.setattr( + gmail_mod, "load_config", lambda *a, **k: gmail_mod.GmailConfig() + ) + client = GmailProvider().build_client( + { + "access_token": "tok", + "refresh_token": "ref", + "token_expiry": time.time() + 3600, + "client_id": "cid", + "client_secret": "cs", + "email": "me@x.com", + }, + lambda d: None, + ) + received, client._message_callback = _collect_callback() + asyncio.run(client._fetch_and_dispatch("m1")) + assert len(received) == 1 + return received[0] + + +_GMAIL_HEADERS = [ + {"name": "From", "value": "Joe "}, + {"name": "Subject", "value": "Top 10 users"}, +] + + +def test_gmail_listener_forwards_real_body_not_snippet(monkeypatch): + # The #444 email: an apostrophe that the escaped snippet leaked as '. + body = "Joe here with an email that's automated. You're probably using Stripe." + msg = _gmail_client( + monkeypatch, + { + "headers": _GMAIL_HEADERS, + "parts": [ + {"mimeType": "text/plain", "body": {"data": _b64(body)}}, + {"mimeType": "text/html", "body": {"data": _b64("

x

")}}, + ], + }, + ) + assert msg.text == f"Subject: Top 10 users\n{body}" + assert msg.truncated is False + + +def test_gmail_listener_converts_html_only_mail(monkeypatch): + msg = _gmail_client( + monkeypatch, + { + "headers": _GMAIL_HEADERS, + "mimeType": "text/html", + "body": {"data": _b64("

Tom & Jerry

")}, + }, + ) + assert msg.text == "Subject: Top 10 users\nTom & Jerry" + + +def test_gmail_listener_caps_long_body_and_flags_it(monkeypatch): + long_body = "word " * 1000 # 5000 chars + msg = _gmail_client( + monkeypatch, + { + "headers": _GMAIL_HEADERS, + "parts": [{"mimeType": "text/plain", "body": {"data": _b64(long_body)}}], + }, + ) + assert msg.truncated is True + body = msg.text.split("\n", 1)[1] + assert len(body) <= gmail_mod._INBOUND_BODY_CHARS + + +# ── Outlook ───────────────────────────────────────────────────────────── + + +def _outlook_dispatch(content): + client = OutlookProvider().build_client( + { + "access_token": "tok", + "refresh_token": "ref", + "token_expiry": time.time() + 3600, + "client_id": "cid", + "email": "me@o.com", + }, + lambda d: None, + ) + received, client._message_callback = _collect_callback() + msg = { + "id": "om1", + "from": {"emailAddress": {"address": "bob@x.com", "name": "Bob"}}, + "subject": "Yo", + "body": {"contentType": "text", "content": content}, + "receivedDateTime": "2026-08-12T10:00:00Z", + } + asyncio.run(client._dispatch_message(msg)) + assert len(received) == 1 + return received[0] + + +def test_outlook_forwards_full_text_body(): + msg = _outlook_dispatch("x" * 300) # past the old 255-char bodyPreview + assert msg.text == "Subject: Yo\n" + "x" * 300 + assert msg.truncated is False + + +def test_outlook_caps_long_body_and_flags_it(): + msg = _outlook_dispatch("word " * 1000) + assert msg.truncated is True + assert len(msg.text.split("\n", 1)[1]) <= outlook_mod._INBOUND_BODY_CHARS + + +# ── Slack ─────────────────────────────────────────────────────────────── + +_NAMES = {"U1": "Ada"} + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("hi <@U1>", "hi @Ada"), + ("hi <@U9>", "hi @U9"), # unresolvable → id + ("hi <@U9|bob>", "hi @bob"), # unresolvable → label + ("see <#C1|general>", "see #general"), + ("see <#C1>", "see #C1"), + ("", "docs (https://x.test)"), + ("", "https://x.test"), + ("", "a@b.co"), + (" ", "@here @channel @everyone"), + ("", "@here"), + ("", "@devs"), + ("", "Nov 14"), + ("Tom & Jerry <3", "Tom & Jerry <3"), + # A literal "<@U1>" the user typed arrives escaped and must stay text. + ("<@U1>", "<@U1>"), + ("", ""), + ], +) +def test_slack_to_plain_text(raw, expected): + assert to_plain_text(raw, lambda uid: _NAMES.get(uid, "")) == expected + + +def test_slack_to_plain_text_survives_resolver_error(): + def boom(uid): + raise RuntimeError("api down") + + assert to_plain_text("hi <@U1>", boom) == "hi @U1" + + +def test_slack_display_name_is_memoized(): + from craftos_integrations.providers.slack.client import SlackClient + + client = SlackClient() + calls = [] + + def fake_user_info(uid): + calls.append(uid) + return {"ok": True, "user": {"profile": {"display_name": "Ada"}}} + + client.get_user_info = fake_user_info + assert client._display_name("U1") == "Ada" + assert client._display_name("U1") == "Ada" + assert calls == ["U1"] + + +def test_slack_display_name_does_not_cache_failures(): + from craftos_integrations.providers.slack.client import SlackClient + + client = SlackClient() + calls = [] + + def flaky(uid): + calls.append(uid) + return {"ok": False} + + client.get_user_info = flaky + assert client._display_name("U1") == "" + assert client._display_name("U1") == "" + assert len(calls) == 2 + + +# ── Twitter ───────────────────────────────────────────────────────────── + + +def test_twitter_decodes_text_before_tag_match(): + from craftos_integrations.providers.twitter.client import ( + TwitterClient, + TwitterConfig, + ) + + client = TwitterClient() + client._config = lambda: TwitterConfig(watch_tag="R&D") + received, client._message_callback = _collect_callback() + tweet = {"id": "1", "author_id": "a1", "text": "R&D check this <now>"} + users = {"a1": {"username": "ada", "name": "Ada"}} + asyncio.run(client._dispatch_mention(tweet, users)) + + assert len(received) == 1 + assert "&" not in received[0].text + assert "" in received[0].text + + +# ── payload contract ──────────────────────────────────────────────────── + + +def test_payload_forwards_truncated(): + msg = PlatformMessage(platform="gmail", sender_id="a", text="x", truncated=True) + assert platform_message_payload(msg)["truncated"] is True + assert platform_message_payload(PlatformMessage("gmail", "a"))["truncated"] is False + + +def test_payload_tolerates_legacy_message_without_truncated(): + class OldMessage: + platform = "slack" + sender_id = "u" + sender_name = "" + text = "hi" + channel_id = "" + channel_name = "" + message_id = "" + raw = {} + + assert platform_message_payload(OldMessage())["truncated"] is False + + +# ── host: the chat sees the cut, the agent gets the facts ──────────────── + + +def _ingest(payload): + """Drive AgentBase._handle_external_event; return the chat payload.""" + from app.agent_base import AgentBase + + captured = [] + + async def fake_chat(p): + captured.append(p) + + agent = SimpleNamespace(_handle_chat_message=fake_chat) + asyncio.run(AgentBase._handle_external_event(agent, payload)) + assert len(captured) == 1 + return captured[0] + + +_EMAIL_EVENT = { + "source": "Gmail", + "integrationType": "gmail", + "contactId": "joe@posthog.com", + "contactName": "Joe", + "messageBody": "Subject: Hi\nYou probably also", + "messageId": "m1", +} + + +def test_truncated_body_marks_cut_for_chat_and_gives_agent_facts(): + chat = _ingest({**_EMAIL_EVENT, "truncated": True}) + # Chat details: the text and a visible cut — no agent instructions. + assert chat["message_body"] == "Subject: Hi\nYou probably also…" + # Agent prompt: the same text plus the facts to fetch the rest. + assert "You probably also…" in chat["text"] + assert "(Body truncated. Message ID: m1)" in chat["text"] + + +def test_untruncated_body_is_unchanged(): + chat = _ingest(_EMAIL_EVENT) + assert chat["message_body"] == "Subject: Hi\nYou probably also" + assert "truncated" not in chat["text"].lower() diff --git a/tests/integrations/test_provider_listeners.py b/tests/integrations/test_provider_listeners.py index 8d7fe514..58849d02 100644 --- a/tests/integrations/test_provider_listeners.py +++ b/tests/integrations/test_provider_listeners.py @@ -68,21 +68,21 @@ async def emit(event): GMAIL_MESSAGE = { "id": "m1", "threadId": "t1", - "snippet": "hello there", "payload": { "headers": [ {"name": "From", "value": "Alice "}, {"name": "Subject", "value": "Hi"}, {"name": "Date", "value": "Tue, 11 Aug 2026 10:00:00 +0000"}, ], - # fields-mask shape: parts skeleton only, no body.data. The + # fields-mask shape: parts skeleton with inline text bodies only + # (attachment parts carry an attachmentId, never data). The # nameless attachmentId part is an inline image — not reported. "parts": [ { "partId": "0", "mimeType": "text/plain", "filename": "", - "body": {"size": 20}, + "body": {"size": 11, "data": "aGVsbG8gdGhlcmU="}, # "hello there" }, { "partId": "1", @@ -178,6 +178,7 @@ async def scenario(): "extra": {"message_id": "m1"}, } ], + "truncated": False, } ] assert cursor == {"history_id": "101", "seen_ids": ["m1"]} @@ -235,7 +236,7 @@ async def scenario(): "id": "om1", "from": {"emailAddress": {"address": "bob@x.com", "name": "Bob"}}, "subject": "Yo", - "bodyPreview": "preview text", + "body": {"contentType": "text", "content": "preview text"}, "receivedDateTime": "2026-08-12T10:00:00Z", "conversationId": "conv1", } @@ -296,6 +297,7 @@ async def scenario(): "is_self_message": False, "raw": OUTLOOK_MESSAGE, "attachments": [], + "truncated": False, } ] # Watermark advanced to the newest receivedDateTime; dedup ids kept. @@ -451,6 +453,7 @@ async def scenario(): "is_self_message": False, "raw": message, "attachments": [], + "truncated": False, } ] assert cursor == {"last_timestamps": {"C1": msg_ts}} From 3a0089df0eafdfa3b1636ce10f91c6dfabaa10e1 Mon Sep 17 00:00:00 2001 From: CraftBot Date: Wed, 30 Sep 2026 19:33:46 +0900 Subject: [PATCH 02/55] Fix posthog proxy destination issue and agent app bidirectional communication issue --- agent-app/docs/agent-guide.md | 1 + agent-app/tools/src/commands/create.ts | 8 ++- agent-app/tools/src/commands/ops.ts | 30 ++++++-- agent-app/tools/src/lib/os-adapter.ts | 26 ++++--- agent-app/tools/src/lib/project.ts | 75 ++++++++++++++++++-- agent_file_system/AGENT.md | 6 +- app/agent_app/agent_view.py | 52 +++++++++++++- app/agent_app/integration_bridge.py | 4 ++ app/agent_app/runner.py | 45 ++++++++++-- app/agent_app/test_data_safety.py | 55 ++++++++++++++ app/agent_base.py | 30 +++++++- app/data/agent_file_system_template/AGENT.md | 6 +- 12 files changed, 300 insertions(+), 38 deletions(-) diff --git a/agent-app/docs/agent-guide.md b/agent-app/docs/agent-guide.md index f62e07e5..b2989b98 100644 --- a/agent-app/docs/agent-guide.md +++ b/agent-app/docs/agent-guide.md @@ -175,6 +175,7 @@ Use the CLI — it resolves the port, authenticates, and validates params: ``` agent-app ops # what can this app do? +agent-app ops # one area's ops, every param explained agent-app run --param value # execute a declared op agent-app data list --filter '...' --limit 20 agent-app data create --json '{...}' diff --git a/agent-app/tools/src/commands/create.ts b/agent-app/tools/src/commands/create.ts index 113026fd..88e70f01 100644 --- a/agent-app/tools/src/commands/create.ts +++ b/agent-app/tools/src/commands/create.ts @@ -24,19 +24,23 @@ async function bootstrapSuperuser(projectDir: string): Promise { const email = 'agent@agent-app.local'; const password = randomBytes(18).toString('base64url'); + // Flags first, then `--`, then email and password: a base64url password + // can start with "-", which PocketBase's CLI would otherwise parse as an + // unknown flag (it then saves nothing and still exits 0). execFileSync( pbBin, [ 'superuser', 'upsert', - email, - password, '--dir', join(pbDir, 'pb_data'), '--migrationsDir', join(pbDir, 'pb_migrations'), '--hooksDir', join(pbDir, 'pb_hooks'), + '--', + email, + password, ], { stdio: 'pipe' }, ); diff --git a/agent-app/tools/src/commands/ops.ts b/agent-app/tools/src/commands/ops.ts index ae6392b7..cb92aff7 100644 --- a/agent-app/tools/src/commands/ops.ts +++ b/agent-app/tools/src/commands/ops.ts @@ -1,31 +1,49 @@ /** - * agent-app ops — the app's declared verb surface (spec O1/O4). - * The agent-facing capability card: what this app can DO. + * agent-app ops [area]: the app's declared verb surface (spec O1/O4). + * The agent-facing capability card: what this app can DO. With an area (an op + * name or the part before its last dot, e.g. `notes` or `notes.create`) it lists + * only those ops, each param with its description, so an agent can look up + * one area of a large app without reading every op. */ import { log } from '../lib/log.ts'; import { loadOps, loadProject } from '../lib/project.ts'; export async function run(args: string[]): Promise { - const dirArg = args.find((a) => !a.startsWith('--')); + const [dirArg, area] = args.filter((a) => !a.startsWith('--')); if (dirArg === undefined) { - log.error('Usage: agent-app ops '); + log.error('Usage: agent-app ops [area]'); return 1; } const project = loadProject(dirArg); - const ops = loadOps(project); + const all = loadOps(project); + const ops = area === undefined ? all : all.filter((o) => o.name === area || o.name.startsWith(area + '.')); + if (ops.length === 0) { + const areas = [...new Set(all.map((o) => o.name.slice(0, Math.max(o.name.lastIndexOf('.'), 0)) || o.name))]; + log.error(`No op matches "${area}". Areas: ${areas.join(', ')}`); + return 1; + } log.raw(`${project.name} (${project.id}) — ${project.baseUrl}\n`); for (const op of ops) { const params = Object.entries(op.params ?? {}) - .map(([k, v]) => `--${k} <${v.type}>${v.required ? '' : '?'}`) + .map(([k, v]) => `--${k} <${v.enum ? v.enum.join('|') : v.type}>${v.required ? '' : '?'}`) .join(' '); const flags = [op.system ? 'system' : '', op.destructive ? 'DESTRUCTIVE' : ''] .filter(Boolean) .join(', '); log.raw(` ${op.name}${params ? ' ' + params : ''}${flags ? ` [${flags}]` : ''}`); log.raw(` ${op.description}`); + if (area !== undefined) { + for (const [k, v] of Object.entries(op.params ?? {})) { + if (v.description) log.raw(` --${k}: ${v.description}`); + } + } } log.raw(`\nRun one: agent-app run ${dirArg} [--param value ...]`); + const example = ops.find((o) => !o.system)?.name; + if (area === undefined && example !== undefined) { + log.raw(`Details: agent-app ops ${dirArg} (each param explained, e.g. ${example.slice(0, example.lastIndexOf('.'))})`); + } log.raw(`Data: agent-app data ${dirArg} [list|get |create|update |delete ] [--json '{...}'] [--filter '...']`); return 0; } diff --git a/agent-app/tools/src/lib/os-adapter.ts b/agent-app/tools/src/lib/os-adapter.ts index 03dfac20..e75ba56d 100644 --- a/agent-app/tools/src/lib/os-adapter.ts +++ b/agent-app/tools/src/lib/os-adapter.ts @@ -21,21 +21,31 @@ export class OSAdapter { this.arch = arch === 'arm64' ? 'arm64' : 'amd64'; } + /** Per-user cache root of the agent-app tooling. */ + private cacheRoot(): string { + return { + darwin: join(homedir(), 'Library', 'Caches', 'craftos-agent-app'), + linux: join(homedir(), '.cache', 'craftos-agent-app'), + win32: join(process.env['LOCALAPPDATA'] ?? join(homedir(), 'AppData', 'Local'), 'craftos-agent-app'), + }[this.platform]; + } + /** Central per-host PocketBase binary cache, versioned (spec B1). */ pbCacheDir(version: string): string { - const override = process.env['AGENT_APP_PB_CACHE']; - const base = - override ?? - { - darwin: join(homedir(), 'Library', 'Caches', 'craftos-agent-app', 'pb'), - linux: join(homedir(), '.cache', 'craftos-agent-app', 'pb'), - win32: join(process.env['LOCALAPPDATA'] ?? join(homedir(), 'AppData', 'Local'), 'craftos-agent-app', 'pb'), - }[this.platform]; + const base = process.env['AGENT_APP_PB_CACHE'] ?? join(this.cacheRoot(), 'pb'); const dir = join(base, version); mkdirSync(dir, { recursive: true }); return dir; } + /** Where the CLI keeps each app's superuser session between commands. Kept + * out of the project folder so it never travels with an exported app. */ + cliSessionDir(): string { + const dir = join(this.cacheRoot(), 'cli-sessions'); + mkdirSync(dir, { recursive: true }); + return dir; + } + /** GitHub release asset name for the pinned version. */ pbAssetName(version: string): string { const os = { darwin: 'darwin', linux: 'linux', win32: 'windows' }[this.platform]; diff --git a/agent-app/tools/src/lib/project.ts b/agent-app/tools/src/lib/project.ts index 14b7561b..87cb59b6 100644 --- a/agent-app/tools/src/lib/project.ts +++ b/agent-app/tools/src/lib/project.ts @@ -1,7 +1,9 @@ /** Operate-command helpers: resolve a project, its port, ops, and auth. */ -import { existsSync, readFileSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; +import { osAdapter } from './os-adapter.ts'; + export interface ProjectRef { dir: string; name: string; @@ -84,7 +86,7 @@ export interface Operation { description: string; system?: boolean; destructive?: boolean; - params?: Record; + params?: Record; executor: { type: string; method?: string; path?: string; collection?: string; action?: string }; } @@ -104,17 +106,72 @@ export function readAgentToken(project: ProjectRef): string | null { } /** Superuser token via the project-local .superuser file (absent on imports). */ -export async function authToken(project: ProjectRef): Promise { +/** + * Superuser session for projects that have a `.superuser` file. A failed + * sign-in is an error, never a silent fallback: carrying on unauthenticated + * turned protected reads into a bare "not found", which agents took to mean + * "already handled by someone else" (IP Manager, 2026-09-29). + */ +export async function authToken(project: ProjectRef, fresh = false): Promise { const credFile = join(project.dir, '.superuser'); if (!existsSync(credFile)) return null; + if (!fresh) { + const cached = cachedSession(project); + if (cached !== null) return cached; + } const { email, password } = JSON.parse(readFileSync(credFile, 'utf8')); const res = await fetch(`${project.baseUrl}/api/collections/_superusers/auth-with-password`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ identity: email, password }), }); - if (!res.ok) return null; - return ((await res.json()) as { token: string }).token; + if (res.status === 429) { + throw new Error( + `Could not sign in to ${project.name}: PocketBase rate-limits sign-ins (HTTP 429, too many in the ` + + `last minute), so nothing was read or changed. Wait a minute, then retry.`, + ); + } + if (!res.ok) { + throw new Error( + `Could not sign in to ${project.name} as its superuser (HTTP ${res.status}): the credentials in ` + + `.superuser do not match the app's database, so nothing was read or changed. ` + + `Relaunch the app from CraftBot to re-apply them, then retry.`, + ); + } + const token = ((await res.json()) as { token: string }).token; + saveSession(project, token); + return token; +} + +/** + * The superuser session is cached per app instance (id + port, so a shadow + * keeps its own) until shortly before it expires. Every CLI command is a new + * process, so without the cache each one signed in again, and PocketBase + * allows 30 sign-ins a minute: an agent working through a batch (filing 30 + * SOPs, say) started failing mid-way. + */ +function sessionFile(project: ProjectRef): string { + return join(osAdapter.cliSessionDir(), `${project.id || 'app'}-${project.port}.json`); +} + +function cachedSession(project: ProjectRef): string | null { + try { + const { token } = JSON.parse(readFileSync(sessionFile(project), 'utf8')) as { token: string }; + const payload = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString('utf8')) as { + exp?: number; + }; + return typeof payload.exp === 'number' && payload.exp * 1000 > Date.now() + 60_000 ? token : null; + } catch { + return null; + } +} + +function saveSession(project: ProjectRef, token: string): void { + try { + writeFileSync(sessionFile(project), JSON.stringify({ token }), { mode: 0o600 }); + } catch { + /* no cache: the next command signs in again */ + } } export async function request( @@ -151,6 +208,12 @@ export async function request( if (extraHeaders !== undefined) Object.assign(headers, extraHeaders); const init: RequestInit = { method, headers }; if (body !== undefined) init.body = JSON.stringify(body); - const res = await fetch(`${project.baseUrl}${path}`, init); + let res = await fetch(`${project.baseUrl}${path}`, init); + if (res.status === 401 && token !== null) { + // A cached session the app no longer accepts (app data reset, secret + // rotated): sign in once more and repeat. A 401 means nothing was done. + headers['Authorization'] = (await authToken(project, true)) ?? ''; + res = await fetch(`${project.baseUrl}${path}`, init); + } return { status: res.status, body: await res.text() }; } diff --git a/agent_file_system/AGENT.md b/agent_file_system/AGENT.md index 11674af6..7698f9e3 100644 --- a/agent_file_system/AGENT.md +++ b/agent_file_system/AGENT.md @@ -1283,13 +1283,13 @@ agent_app_convert(source, ...) Rebuild a foreign app as a Agent App Read/write a project's live data with the agent-app CLI via `run_shell` (absolute paths required): ``` +node /agent-app/tools/src/cli.ts run --param value +node /agent-app/tools/src/cli.ts ops [area] node /agent-app/tools/src/cli.ts data schema node /agent-app/tools/src/cli.ts data list|create|update|delete ... -node /agent-app/tools/src/cli.ts run --param value -node /agent-app/tools/src/cli.ts ops ``` -`agent_app_usage(project_id)` returns the exact commands for a given project. Use `agent_app_http` only when the CLI cannot do it. While a code change is in progress, agent writes are routed to the dev instance — test writes to an app's real data are refused. +Prefer a declared operation (`run`) over a raw `data` write: an operation runs the app's own rules (conversions, cascades, file and workbook write-backs), while `data` writes the record as-is. `ops ` explains every param of one area (e.g. `notes`), including allowed values; operations take dates as YYYY-MM-DD. `agent_app_usage(project_id)` returns the exact commands for a given project, with its operations listed. Use `agent_app_http` only when the CLI cannot do it. While a code change is in progress, agent writes are routed to the dev instance — test writes to an app's real data are refused. ### Build / delivery lifecycle (one flow for builds and modifies) diff --git a/app/agent_app/agent_view.py b/app/agent_app/agent_view.py index ee4551ae..7ea2becf 100644 --- a/app/agent_app/agent_view.py +++ b/app/agent_app/agent_view.py @@ -2,8 +2,9 @@ """ What the agent and the user each SEE of a Agent App. -`schema_block()` and `capability_block()` inline the app's data model and its -callable operations into the agent's prompt. Advisory pointers do not work on +`schema_block()`, `operations_block()` and `capability_block()` inline the +app's data model, its declared operations and the integrations it can reach +into the agent's prompt. Advisory pointers do not work on weak models: across two recorded incidents the agent ignored "Read AGENT_APP.md", never ran `agent-app ops`, and guessed collection names instead (`items`, `tasks`). It cannot ignore what is already in its context. They read @@ -106,6 +107,53 @@ def schema_block(base_url: str, max_chars: int = 2000) -> Optional[str]: return block +def operations_block(base_url: str, max_chars: int = 6000) -> Optional[str]: + """The app's declared operations, grouped by area, one line per area. + + Same reason as the schema: an app whose operations were only reachable by + running `agent-app ops` got driven through raw `data` writes instead, which + skip every rule the app's own routes apply (currency conversion, cascades, + workbook write-backs). Each verb shows its required params in parentheses + and a trailing `!` when it is destructive, e.g. + notes: list, get(name), create(title), update(name), delete(name)! + """ + described = _describe(base_url) + if not described: + return None + groups: Dict[str, List[tuple]] = {} + for op in described.get("operations") or []: + if not isinstance(op, dict) or op.get("system"): + continue + name = str(op.get("name") or "") + area, _, verb = name.rpartition(".") + if not area: + area, verb = name, name + params = op.get("params") or {} + required = [ + k for k, spec in params.items() if isinstance(spec, dict) and spec.get("required") + ] + groups.setdefault(area, []).append((verb, required, bool(op.get("destructive")))) + if not groups: + return None + + def render(with_params: bool) -> str: + lines = [] + for area, verbs in groups.items(): + labels = [ + verb + + (f"({', '.join(req)})" if with_params and req else "") + + ("!" if destructive else "") + for verb, req, destructive in verbs + ] + lines.append(f" {area}: {', '.join(labels)}") + return "\n".join(lines) + + block = render(True) + if len(block) > max_chars: # very large apps: verbs only + block = render(False) + return block + + _CAP_CACHE: Dict[str, tuple] = {} _CAP_TTL_SECONDS = 300 diff --git a/app/agent_app/integration_bridge.py b/app/agent_app/integration_bridge.py index 1abdcdf5..3bdaf1c6 100644 --- a/app/agent_app/integration_bridge.py +++ b/app/agent_app/integration_bridge.py @@ -694,6 +694,10 @@ def _validate_token(self, request: web.Request) -> Optional[str]: "slack": ("https://slack.com", ("slack.com",)), "discord": ("https://discord.com", ("discord.com", "discordapp.com")), "notion": ("https://api.notion.com", ("api.notion.com",)), + # PostHog's host lives on the credential (US or EU cloud); the suffix + # match admits both. Self-hosted installs are denied for raw proxying, + # while PostHog actions still resolve their own host. + "posthog": ("https://us.posthog.com", ("posthog.com",)), "hubspot": ("https://api.hubapi.com", ("api.hubapi.com",)), "jira": ("https://api.atlassian.com", ("atlassian.net", "api.atlassian.com")), "linkedin": ("https://api.linkedin.com", ("api.linkedin.com",)), diff --git a/app/agent_app/runner.py b/app/agent_app/runner.py index 471f49c4..5300d183 100644 --- a/app/agent_app/runner.py +++ b/app/agent_app/runner.py @@ -77,6 +77,27 @@ def read_superuser_creds(project_dir: Path): return None +def _superuser_stamp(db_path: Path, email: str) -> Optional[str]: + """The `updated` stamp of the superuser row for `email` in a PocketBase + data.db, or None when the database or the row does not exist. Read-only; + used to confirm that `superuser upsert` really saved.""" + import sqlite3 + + if not Path(db_path).exists(): + return None + try: + con = sqlite3.connect(Path(db_path).resolve().as_uri() + "?mode=ro", uri=True) + try: + row = con.execute( + "SELECT updated FROM _superusers WHERE email = ?", (email,) + ).fetchone() + finally: + con.close() + except sqlite3.Error: + return None + return str(row[0]) if row else None + + class AgentAppRunner: """Drives Agent App projects through scaffold → install → gate → serve.""" @@ -355,34 +376,48 @@ async def ensure_superuser( creds = read_superuser_creds(project_dir) email = creds[0] if creds else "agent@agent-app.local" password = creds[1] if creds else secrets.token_urlsafe(18) - + db_path = (data_dir if data_dir is not None else pb_dir / "pb_data") / "data.db" + stamp_before = _superuser_stamp(db_path, email) + + # Flags first, then `--`, then the positional email and password: a + # token_urlsafe password can start with "-", and without the + # separator PocketBase's CLI parses it as an unknown flag, prints an + # error, saves NOTHING and still exits 0 (IP Manager 2026-09-29: the + # .superuser file and the database disagreed from then on, every + # CLI read ran unauthenticated and came back "not found"). code, out = await self._run( [ str(pb_bin), "superuser", "upsert", - email, - password, "--dir", str(data_dir if data_dir is not None else pb_dir / "pb_data"), "--migrationsDir", str(pb_dir / "pb_migrations"), "--hooksDir", str(pb_dir / "pb_hooks"), + "--", + email, + password, ], timeout=60, ) - if code != 0: + # PocketBase's CLI exits 0 even when a command fails, so the exit + # code alone proves nothing: the save is confirmed from the database + # itself (the superuser row exists and its `updated` stamp moved). + saved = code == 0 and _superuser_stamp(db_path, email) not in (None, stamp_before) + if not saved: # FAIL CLOSED. A serve without a superuser makes PocketBase pop # the user's SYSTEM BROWSER with a one-time /_/#/pbinstall/… # admin-installer token (observed live 2026-08-05, kanban_board # marketplace install) — a jarring tab AND an unauthenticated # admin takeover link. A failed launch with evidence beats that. # Never include the password in the error. + tail = out[-300:].replace(password, "") raise RuntimeError( f"superuser upsert failed for {project_dir.name} — refusing " f"to serve without one (PocketBase would open its installer " - f"page in the user's browser): {out[-300:]}" + f"page in the user's browser): {tail}" ) try: cred_file.write_text( diff --git a/app/agent_app/test_data_safety.py b/app/agent_app/test_data_safety.py index 872013fc..2492fa9e 100644 --- a/app/agent_app/test_data_safety.py +++ b/app/agent_app/test_data_safety.py @@ -1737,6 +1737,61 @@ async def _fake_run(cmd, timeout, cwd=None): print("§23 superuser fail-closed: OK") +# ── §23b a silent upsert failure (exit 0, nothing saved) also fails closed ── +# PocketBase's CLI exits 0 even when a command errors. A password starting +# with "-" was parsed as a flag: nothing saved, launch "succeeded", and every +# CLI read ran unauthenticated (IP Manager, 2026-09-29). The save is now +# confirmed from the database, and the password always follows `--`. +import sqlite3 as _sqlite3 + +with tempfile.TemporaryDirectory() as tmp: + runner23b = AgentAppRunner(Path(tmp)) + proj23b = Path(tmp) / "proj" + (proj23b / "pb" / "pb_data").mkdir(parents=True) + (proj23b / ".superuser").write_text( + _json.dumps({"email": "agent@agent-app.local", "password": "-Starts-with-a-dash"}) + "\n", + encoding="utf-8", + ) + seen_cmds = [] + + async def _fake_pb_binary23b(): + return Path("pocketbase") + + async def _silent_fail(cmd, timeout, cwd=None): + seen_cmds.append(cmd) + return (0, "Error: unknown shorthand flag: 'S' in -Starts-with-a-dash") + + runner23b.pb_binary = _fake_pb_binary23b + runner23b._run = _silent_fail + try: + asyncio.run(runner23b.ensure_superuser(proj23b)) + raise AssertionError("an upsert that saved nothing must refuse to serve") + except RuntimeError as e: + assert "refusing to serve" in str(e) + assert "-Starts-with-a-dash" not in str(e), "the password must never appear in the error" + + cmd = seen_cmds[-1] + sep = cmd.index("--") + assert cmd[sep + 1 :] == ["agent@agent-app.local", "-Starts-with-a-dash"], cmd + assert all(not str(a).startswith("--dir") or i < sep for i, a in enumerate(cmd)) + + async def _real_save(cmd, timeout, cwd=None): + db = proj23b / "pb" / "pb_data" / "data.db" + con = _sqlite3.connect(db) + con.execute("CREATE TABLE IF NOT EXISTS _superusers (email TEXT, updated TEXT)") + con.execute("DELETE FROM _superusers") + con.execute("INSERT INTO _superusers VALUES (?, ?)", ("agent@agent-app.local", "2026-09-29 01:00:00.000Z")) + con.commit() + con.close() + return (0, 'Successfully saved superuser "agent@agent-app.local"!') + + runner23b._run = _real_save + asyncio.run(runner23b.ensure_superuser(proj23b)) + stored = _json.loads((proj23b / ".superuser").read_text(encoding="utf-8")) + assert stored["password"] == "-Starts-with-a-dash" +print("§23b superuser silent failure fails closed, password after --: OK") + + # ── §24 the heartbeat never loses a wakeup (stale build) ─────────────────── # The old deferred-wakeup bug: a suppressed redispatch with nothing left to # re-fire it left a build stale at 'fixing' forever. The supervisor's diff --git a/app/agent_base.py b/app/agent_base.py index 96c2aea4..e6157d20 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -2239,6 +2239,28 @@ def _build_agent_app_note(agent_app_project_id: str) -> str: if schema else f"Data model: run node {_agent_app_cli} data {proj.path} schema\n" ) + # Operations go in the prompt too: an app whose verbs were + # only discoverable by running `agent-app ops` got driven + # through raw `data` writes, which skip the rules the app's + # own routes apply (conversions, cascades, write-backs). + ops = None + try: + from app.agent_app.agent_view import operations_block + + base = proj.backend_url or proj.url + if base: + ops = operations_block(base.rstrip("/")) + except Exception: + ops = None + operations = ( + f"Operations: USE THESE FIRST. They apply the app's own rules, which a raw\n" + f"`data` write skips. (x) = required params, ! = destructive (confirm with\n" + f"the user first):\n{ops}\n" + f" Every param with its allowed values, for one area:\n" + f" node {_agent_app_cli} ops {proj.path} \n" + if ops + else "" + ) # Same principle as the schema: capabilities go IN the # prompt. Three builds stubbed the user's email feature # around an invented SMTP requirement because nothing in @@ -2255,17 +2277,19 @@ def _build_agent_app_note(agent_app_project_id: str) -> str: return ( f"[INTERACTING WITH AGENT APP: {proj.name} ({agent_app_project_id})]\n" f"Project path: {proj.path}\n" + f"{operations}" f"{model}" f"{caps}" - f"Values: dates as ISO or 'tomorrow'/'next monday' (the CLI resolves them);\n" - f'references by name, e.g. --list "To Do". \n' + f"Values: `data` takes dates as ISO or 'tomorrow'/'next monday' (the CLI resolves\n" + f'them) and references by name, e.g. --list "To Do". Operations (`run`) take\n' + f"dates as YYYY-MM-DD only.\n" f"To OPERATE the app, use the agent-app CLI via run_shell with ABSOLUTE paths\n" f"(the shell's cwd is NOT the repo root):\n" + f' node {_agent_app_cli} run {proj.path} --param "value"\n' f' node {_agent_app_cli} data {proj.path} create --field "value"\n' f" ALWAYS quote values — an unquoted # starts a shell comment and\n" f" silently drops the rest of the command.\n" f" node {_agent_app_cli} data {proj.path} list --limit 20\n" - f" node {_agent_app_cli} run {proj.path} --param value\n" f"If debugging, read {proj.path}/logs/pocketbase.log and logs/frontend_console.log.\n" f"Using the app needs no skill. To CHANGE its code, or import/diagnose one,\n" f"load the right Agent App skill first (use_skill); list_skills shows all skills.\n" diff --git a/app/data/agent_file_system_template/AGENT.md b/app/data/agent_file_system_template/AGENT.md index 11674af6..7698f9e3 100644 --- a/app/data/agent_file_system_template/AGENT.md +++ b/app/data/agent_file_system_template/AGENT.md @@ -1283,13 +1283,13 @@ agent_app_convert(source, ...) Rebuild a foreign app as a Agent App Read/write a project's live data with the agent-app CLI via `run_shell` (absolute paths required): ``` +node /agent-app/tools/src/cli.ts run --param value +node /agent-app/tools/src/cli.ts ops [area] node /agent-app/tools/src/cli.ts data schema node /agent-app/tools/src/cli.ts data list|create|update|delete ... -node /agent-app/tools/src/cli.ts run --param value -node /agent-app/tools/src/cli.ts ops ``` -`agent_app_usage(project_id)` returns the exact commands for a given project. Use `agent_app_http` only when the CLI cannot do it. While a code change is in progress, agent writes are routed to the dev instance — test writes to an app's real data are refused. +Prefer a declared operation (`run`) over a raw `data` write: an operation runs the app's own rules (conversions, cascades, file and workbook write-backs), while `data` writes the record as-is. `ops ` explains every param of one area (e.g. `notes`), including allowed values; operations take dates as YYYY-MM-DD. `agent_app_usage(project_id)` returns the exact commands for a given project, with its operations listed. Use `agent_app_http` only when the CLI cannot do it. While a code change is in progress, agent writes are routed to the dev instance — test writes to an app's real data are refused. ### Build / delivery lifecycle (one flow for builds and modifies) From 0d19ea02cb61f40da4db59b664cebdcb64c58680 Mon Sep 17 00:00:00 2001 From: ahmad-ajmal Date: Wed, 30 Sep 2026 12:03:04 +0100 Subject: [PATCH 03/55] clean comments --- app/agent_base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/agent_base.py b/app/agent_base.py index 43ab346d..874ff809 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -2450,8 +2450,6 @@ async def _handle_external_event(self, payload: Dict) -> None: # The listener capped a long body (PlatformMessage.truncated): # mark the cut where both the agent and the chat details see it. - # The agent additionally gets the facts to fetch the rest itself - # (docs/plans/inbound-message-fidelity-plan.md). truncated = bool(payload.get("truncated")) and bool(message_body) if truncated: message_body = f"{message_body}…" @@ -2459,7 +2457,7 @@ async def _handle_external_event(self, payload: Dict) -> None: # Normalized attachments (PlatformMessage.attachments) become # descriptor lines with retrieval hints — appended to the body, # or standing in for it on media-only messages so they are no - # longer dropped (docs/plans/attachment-reception-plan.md). + # longer dropped. from app.integrations import format_attachment_descriptors att_lines = format_attachment_descriptors( From e6d0d7f35cd27814ecf7f1f608f133fc1c453e2b Mon Sep 17 00:00:00 2001 From: ahmad-ajmal Date: Wed, 30 Sep 2026 12:08:10 +0100 Subject: [PATCH 04/55] clean up comments --- craftos_integrations/README.md | 2 +- craftos_integrations/base.py | 3 +-- craftos_integrations/helpers/text.py | 3 +-- craftos_integrations/providers/slack/formatting.py | 2 +- tests/integrations/test_listener_text_fidelity.py | 4 ++-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/craftos_integrations/README.md b/craftos_integrations/README.md index d3cbc3e5..f23de060 100644 --- a/craftos_integrations/README.md +++ b/craftos_integrations/README.md @@ -179,7 +179,7 @@ listeners unescape HTML entities and render platform markup before emitting, and never forward an API preview field (Gmail `snippet`, Outlook `bodyPreview`) as the body. A listener that caps the length uses `helpers.clip` and sets `truncated`; the host then marks the cut and tells -the agent the message id (`docs/plans/inbound-message-fidelity-plan.md`). +the agent the message is truncated and how to read the rest by message id. --- diff --git a/craftos_integrations/base.py b/craftos_integrations/base.py index 0a9de71c..dd907af7 100644 --- a/craftos_integrations/base.py +++ b/craftos_integrations/base.py @@ -45,8 +45,7 @@ class PlatformMessage: attachments: List[Dict[str, Any]] = field(default_factory=list) # True when the listener capped ``text`` (``helpers.clip``). The HOST # marks the cut and gives the agent the message id to fetch the rest; - # listeners only declare the fact - # (docs/plans/inbound-message-fidelity-plan.md). + # listeners only declare the fact. truncated: bool = False diff --git a/craftos_integrations/helpers/text.py b/craftos_integrations/helpers/text.py index abb8e7c4..93deb701 100644 --- a/craftos_integrations/helpers/text.py +++ b/craftos_integrations/helpers/text.py @@ -2,8 +2,7 @@ Listeners that cap a body's length (to keep inbound token cost bounded) use ``clip`` so every cap cuts the same way and reports that it cut — -the flag goes to ``PlatformMessage.truncated`` and the host marks the cut -(docs/plans/inbound-message-fidelity-plan.md). +the flag goes to ``PlatformMessage.truncated`` and the host marks the cut. """ from __future__ import annotations diff --git a/craftos_integrations/providers/slack/formatting.py b/craftos_integrations/providers/slack/formatting.py index fd008ad0..369d2c89 100644 --- a/craftos_integrations/providers/slack/formatting.py +++ b/craftos_integrations/providers/slack/formatting.py @@ -3,7 +3,7 @@ Slack's ``text`` field is not what the user typed: ``& < >`` arrive HTML-escaped and every reference is wrapped in angle-bracket markup. Forwarded as-is, the agent and the chat details show ``<@U0123ABC>`` and -``&`` (issue #444; docs/plans/inbound-message-fidelity-plan.md). +``&``. Pure module — the only outside knowledge (user id → display name) is injected as ``resolve_user``, so it is testable without a client. diff --git a/tests/integrations/test_listener_text_fidelity.py b/tests/integrations/test_listener_text_fidelity.py index d627d1ba..15c04a89 100644 --- a/tests/integrations/test_listener_text_fidelity.py +++ b/tests/integrations/test_listener_text_fidelity.py @@ -2,8 +2,8 @@ Listeners emit plain, decoded text built from the real message body, cap it with ``clip`` and declare the cut as ``PlatformMessage.truncated``; the -host marks the cut for the chat and hands the agent the facts to fetch the -rest (docs/plans/inbound-message-fidelity-plan.md). +host marks the cut for the chat and tells the agent the message is +truncated and how to fetch the rest. No pytest-asyncio in this repo — async paths are driven with asyncio.run. """ From dc95e6f35602455e0784529882f6daa1abac4594 Mon Sep 17 00:00:00 2001 From: ahmad-ajmal Date: Wed, 30 Sep 2026 12:11:07 +0100 Subject: [PATCH 05/55] agent recieves truncated information --- app/agent_base.py | 16 +++++++++++++--- .../integrations/test_listener_text_fidelity.py | 10 +++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/app/agent_base.py b/app/agent_base.py index 874ff809..c9a4b4f9 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -2526,11 +2526,21 @@ async def _handle_external_event(self, payload: Dict) -> None: location_parts.append(f"channel {channel_id}") location_str = f" in {' / '.join(location_parts)}" if location_parts else "" - # Facts only: the agent decides whether it needs the full body. + # Tell the agent the body above is partial and how to get the + # rest; whether the rest is needed stays the agent's call. truncation_note = "" if truncated: - id_part = f" Message ID: {message_id}" if message_id else "" - truncation_note = f"(Body truncated.{id_part})\n" + fetch_hint = ( + f" If the rest matters, read the full message with the " + f"{integration_type} action that gets a message by ID " + f"(message ID: {message_id})." + if message_id + else "" + ) + truncation_note = ( + f"[This message is truncated: only the beginning is shown " + f"above.{fetch_hint}]\n" + ) if is_self_message: # Self-message = user is directly talking to the agent via their own platform. diff --git a/tests/integrations/test_listener_text_fidelity.py b/tests/integrations/test_listener_text_fidelity.py index 15c04a89..99e735e5 100644 --- a/tests/integrations/test_listener_text_fidelity.py +++ b/tests/integrations/test_listener_text_fidelity.py @@ -331,7 +331,15 @@ def test_truncated_body_marks_cut_for_chat_and_gives_agent_facts(): assert chat["message_body"] == "Subject: Hi\nYou probably also…" # Agent prompt: the same text plus the facts to fetch the rest. assert "You probably also…" in chat["text"] - assert "(Body truncated. Message ID: m1)" in chat["text"] + assert "[This message is truncated" in chat["text"] + assert "gmail action that gets a message by ID (message ID: m1)" in chat["text"] + + +def test_truncated_body_without_id_still_tells_agent(): + chat = _ingest({**_EMAIL_EVENT, "messageId": "", "truncated": True}) + assert "[This message is truncated: only the beginning is shown above.]" in ( + chat["text"] + ) def test_untruncated_body_is_unchanged(): From 68e3aef021504752a94315950632995ee55a904d Mon Sep 17 00:00:00 2001 From: CraftBot Date: Thu, 1 Oct 2026 11:21:20 +0900 Subject: [PATCH 06/55] Provide LLM reasoning effort param --- agent_core/core/impl/llm/cache/gemini.py | 17 +- agent_core/core/impl/llm/interface.py | 42 +- agent_core/core/impl/llm/reasoning_wire.py | 87 ++ .../impl/llm/transports/anthropic_messages.py | 28 +- .../impl/llm/transports/bedrock_converse.py | 18 + .../impl/llm/transports/chat_completions.py | 23 +- .../core/impl/llm/transports/gemini_native.py | 30 +- agent_core/core/llm/google_gemini_client.py | 46 +- .../models/chatgpt_subscription_client.py | 27 +- agent_core/core/models/factory.py | 15 +- agent_core/core/models/reasoning.py | 814 ++++++++++++++++++ scripts/probe_reasoning.py | 227 +++++ tests/llm/golden/conftest.py | 337 ++++++++ tests/llm/golden/snapshots/anthropic.json | 195 +++++ tests/llm/golden/snapshots/bedrock.json | 245 ++++++ tests/llm/golden/snapshots/byteplus.json | 83 ++ tests/llm/golden/snapshots/deepseek.json | 158 ++++ tests/llm/golden/snapshots/gemini.json | 242 ++++++ tests/llm/golden/snapshots/groq.json | 146 ++++ tests/llm/golden/snapshots/openai.json | 12 +- .../golden/snapshots/openrouter_claude.json | 181 ++++ .../snapshots/openrouter_non_claude.json | 158 ++++ tests/llm/golden/snapshots/remote.json | 72 ++ .../golden/snapshots/unruled_anthropic.json | 183 ++++ .../llm/golden/snapshots/unruled_bedrock.json | 220 +++++ .../llm/golden/snapshots/unruled_gemini.json | 230 +++++ tests/llm/golden/snapshots/unruled_grok.json | 158 ++++ .../llm/golden/snapshots/unruled_openai.json | 154 ++++ .../snapshots/unruled_openrouter_claude.json | 173 ++++ tests/llm/golden/test_golden_payloads.py | 103 ++- tests/llm/test_reasoning_rules.py | 449 ++++++++++ tests/llm/test_reliability.py | 244 ++++++ tests/test_bedrock_token_normalization.py | 3 + 33 files changed, 5069 insertions(+), 51 deletions(-) create mode 100644 agent_core/core/impl/llm/reasoning_wire.py create mode 100644 agent_core/core/models/reasoning.py create mode 100644 scripts/probe_reasoning.py create mode 100644 tests/llm/golden/conftest.py create mode 100644 tests/llm/golden/snapshots/anthropic.json create mode 100644 tests/llm/golden/snapshots/bedrock.json create mode 100644 tests/llm/golden/snapshots/byteplus.json create mode 100644 tests/llm/golden/snapshots/deepseek.json create mode 100644 tests/llm/golden/snapshots/gemini.json create mode 100644 tests/llm/golden/snapshots/groq.json create mode 100644 tests/llm/golden/snapshots/openrouter_claude.json create mode 100644 tests/llm/golden/snapshots/openrouter_non_claude.json create mode 100644 tests/llm/golden/snapshots/remote.json create mode 100644 tests/llm/golden/snapshots/unruled_anthropic.json create mode 100644 tests/llm/golden/snapshots/unruled_bedrock.json create mode 100644 tests/llm/golden/snapshots/unruled_gemini.json create mode 100644 tests/llm/golden/snapshots/unruled_grok.json create mode 100644 tests/llm/golden/snapshots/unruled_openai.json create mode 100644 tests/llm/golden/snapshots/unruled_openrouter_claude.json create mode 100644 tests/llm/test_reasoning_rules.py create mode 100644 tests/llm/test_reliability.py diff --git a/agent_core/core/impl/llm/cache/gemini.py b/agent_core/core/impl/llm/cache/gemini.py index fc06a813..1b1376a9 100644 --- a/agent_core/core/impl/llm/cache/gemini.py +++ b/agent_core/core/impl/llm/cache/gemini.py @@ -10,7 +10,7 @@ import hashlib import logging import time -from typing import Any, Dict, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from .config import get_cache_config @@ -80,6 +80,8 @@ def get_or_create_cache( call_type: str, temperature: float, max_tokens: int, + thinking_budget: Optional[int] = None, + thinking_level: Optional[str] = None, ) -> Dict[str, Any]: """Get response using explicit cache, creating cache if needed. @@ -89,10 +91,19 @@ def get_or_create_cache( call_type: Type of LLM call (e.g., "reasoning", "action_selection"). temperature: Sampling temperature. max_tokens: Maximum output tokens. + thinking_budget: Reasoning token budget (Gemini 2.5), forwarded + to every generation call. + thinking_level: Reasoning level (Gemini 3.x), forwarded to every + generation call. Returns: Response dict with tokens_used, content, cached_tokens, etc. """ + thinking: Dict[str, Any] = { + "thinking_budget": thinking_budget, + "thinking_level": thinking_level, + } + # Check if system prompt is large enough for explicit caching # Gemini requires at least 1024 tokens; skip explicit cache if too small estimated_tokens = self._estimate_tokens(system_prompt) @@ -109,6 +120,7 @@ def get_or_create_cache( temperature=temperature, max_output_tokens=max_tokens, json_mode=True, + **thinking, ) cache_key = self._make_cache_key(system_prompt, call_type) @@ -132,6 +144,7 @@ def get_or_create_cache( temperature=temperature, max_output_tokens=max_tokens, json_mode=True, + **thinking, ) except Exception as e: logger.warning( @@ -166,6 +179,7 @@ def get_or_create_cache( temperature=temperature, max_output_tokens=max_tokens, json_mode=True, + **thinking, ) except Exception as e: logger.warning( @@ -185,6 +199,7 @@ def get_or_create_cache( temperature=temperature, max_output_tokens=max_tokens, json_mode=True, + **thinking, ) def invalidate_cache(self, system_prompt: str, call_type: str) -> None: diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index c5965b9d..7a97fd51 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -42,6 +42,7 @@ RecordLLMCallHook, ) from agent_core.core.impl.llm import transports as _transports +from agent_core.core.models.reasoning import ReasoningDecision, resolve_reasoning from agent_core.core.models.registry import ( get_registry as _get_registry, session_cc_providers as _session_cc_providers, @@ -227,6 +228,9 @@ def __init__( # multi-provider outage terminates instead of nesting # primary -> fb -> fb-of-fb recursion. self._is_fallback_instance = False + # (provider, model, auth_mode) whose reasoning default was last + # logged, so the decision is logged once per model, not per call. + self._reasoning_logged_for: Optional[tuple] = None # Defer imports to avoid circular dependency from app.models.factory import ModelFactory @@ -776,7 +780,8 @@ def _check_context_fits( from app.config import get_context_window window = get_context_window() - budget = window - self.max_tokens + reserved = self._output_reservation() + budget = window - reserved total = count_tokens(system_prompt or "") if messages: @@ -795,10 +800,43 @@ def _check_context_fits( if total > budget: raise LLMContextOverflowError( f"Request of ~{total} input tokens exceeds the {budget}-token budget " - f"({window} window - {self.max_tokens} reserved for output) for " + f"({window} window - {reserved} reserved for output) for " f"{self.provider}/{self.model}." ) + def reasoning_decision(self) -> Optional[ReasoningDecision]: + """Reasoning default for the current provider and model. + + None means the model has no rule in agent_core/core/models/reasoning.py + and its requests carry no reasoning parameter. Resolved on every call + so a model switch (reinitialize) or a fallback interface uses its own + row; logged once per distinct model. + """ + decision = resolve_reasoning(self.provider, self.model, self._auth_mode) + log_key = (self.provider, self.model, self._auth_mode) + if log_key != self._reasoning_logged_for: + self._reasoning_logged_for = log_key + if decision is None: + logger.info( + f"[REASONING] {self.provider}/{self.model}: no reasoning rule " + f"for this model, reasoning parameters not sent" + ) + else: + logger.info(f"[REASONING] {decision.key}: {decision.describe()}") + return decision + + def _output_reservation(self) -> int: + """Output tokens a request reserves out of the context window. + + Requests with a reasoning default carry a larger output cap (reasoning + tokens count against it), so the window check reserves that cap; + otherwise the provider would reject the request for size instead. + """ + decision = self.reasoning_decision() + if decision is None: + return self.max_tokens + return decision.output_cap(self.max_tokens) + def _generate_response_sync( self, system_prompt: Optional[str] = None, diff --git a/agent_core/core/impl/llm/reasoning_wire.py b/agent_core/core/impl/llm/reasoning_wire.py new file mode 100644 index 00000000..9203801b --- /dev/null +++ b/agent_core/core/impl/llm/reasoning_wire.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Render a reasoning default into request fields, one function per transport. + +The table and the level policy live in agent_core/core/models/reasoning.py; +this module only knows how each transport spells a ``ReasoningDecision``. +Callers skip these functions entirely when a model has no rule, so a model +without a rule never gains a field here. + +A decision whose wire the transport cannot express is a bug in the rules +table (a row filed under the wrong provider), so it raises instead of being +dropped silently. tests/llm/test_reasoning_rules.py checks every row against +``TRANSPORT_WIRES`` so this never reaches production. +""" + +from __future__ import annotations + +from typing import Any, Dict, FrozenSet, Mapping, Tuple + +from agent_core.core.models.reasoning import ReasoningDecision, ReasoningWire + +#: Reasoning wires each transport (ProviderProfile.wire) can express. +TRANSPORT_WIRES: Mapping[str, FrozenSet[ReasoningWire]] = { + "chat_completions": frozenset( + { + ReasoningWire.EFFORT, + ReasoningWire.OPENROUTER_EFFORT, + ReasoningWire.OPENROUTER_BUDGET, + } + ), + "anthropic_messages": frozenset( + {ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET} + ), + "bedrock_converse": frozenset( + {ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET} + ), + "gemini_native": frozenset( + {ReasoningWire.GEMINI_LEVEL, ReasoningWire.GEMINI_BUDGET} + ), +} + + +def _unsupported(decision: ReasoningDecision, transport: str) -> ValueError: + return ValueError( + f"Reasoning rule {decision.key!r} uses wire {decision.wire.value!r}, " + f"which the {transport} transport cannot send. The row is filed under " + f"the wrong provider in agent_core/core/models/reasoning.py." + ) + + +def chat_completions_fields( + decision: ReasoningDecision, +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Fields for a Chat Completions request: (top-level kwargs, extra_body).""" + if decision.wire is ReasoningWire.EFFORT: + return {"reasoning_effort": decision.level}, {} + if decision.wire is ReasoningWire.OPENROUTER_EFFORT: + return {}, {"reasoning": {"effort": decision.level}} + if decision.wire is ReasoningWire.OPENROUTER_BUDGET: + return {}, {"reasoning": {"max_tokens": decision.budget_tokens}} + raise _unsupported(decision, "chat_completions") + + +def anthropic_fields(decision: ReasoningDecision) -> Dict[str, Any]: + """Anthropic Messages fields. + + The same dict is the ``additionalModelRequestFields`` of a Bedrock + Converse request for Claude, which forwards these keys to the model. + """ + if decision.wire is ReasoningWire.ANTHROPIC_ADAPTIVE: + return { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": decision.level}, + } + if decision.wire is ReasoningWire.ANTHROPIC_BUDGET: + return { + "thinking": {"type": "enabled", "budget_tokens": decision.budget_tokens} + } + raise _unsupported(decision, "anthropic_messages/bedrock_converse") + + +def gemini_thinking_kwargs(decision: ReasoningDecision) -> Dict[str, Any]: + """Keyword arguments for the GeminiClient text-generation methods.""" + if decision.wire is ReasoningWire.GEMINI_LEVEL: + return {"thinking_level": decision.level} + if decision.wire is ReasoningWire.GEMINI_BUDGET: + return {"thinking_budget": decision.budget_tokens} + raise _unsupported(decision, "gemini_native") diff --git a/agent_core/core/impl/llm/transports/anthropic_messages.py b/agent_core/core/impl/llm/transports/anthropic_messages.py index dd031ead..56639b0f 100644 --- a/agent_core/core/impl/llm/transports/anthropic_messages.py +++ b/agent_core/core/impl/llm/transports/anthropic_messages.py @@ -10,10 +10,14 @@ from typing import Any, Dict, List, Optional from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error from agent_core.utils.logger import logger +# Anthropic requires max_tokens; 16384 (Claude 4 default) avoids truncation. +_DEFAULT_MAX_TOKENS = 16384 + @profile("llm_anthropic_call", OperationCategory.LLM) def generate( @@ -71,11 +75,20 @@ def generate( if not iface._anthropic_client: raise RuntimeError("Anthropic client was not initialised.") - # Build the message - use pre-built messages for multi-turn, or single-turn - # Anthropic requires max_tokens; use 16384 (Claude 4 default) to avoid truncation + # Reasoning default for this exact model (None: no rule, so the + # request is shaped exactly as it was before reasoning defaults). + reasoning = iface.reasoning_decision() + + # Build the message - use pre-built messages for multi-turn, or single-turn. + # Thinking tokens count against max_tokens, so a reasoning default + # raises it (staying below the SDK's non-streaming ceiling). message_kwargs: Dict[str, Any] = { "model": iface.model, - "max_tokens": 16384, + "max_tokens": ( + _DEFAULT_MAX_TOKENS + if reasoning is None + else reasoning.output_cap(_DEFAULT_MAX_TOKENS) + ), "messages": messages if messages is not None else [ @@ -109,7 +122,14 @@ def generate( # Short prompt - use simple string format (no caching) message_kwargs["system"] = system_prompt - message_kwargs["extra_body"] = {"temperature": iface.temperature} + if reasoning is not None: + message_kwargs.update(reasoning_wire.anthropic_fields(reasoning)) + + # Thinking is incompatible with temperature on Claude 4.5/4.6, and + # Claude 4.7+ rejects any non-default temperature, so rows with + # reasoning drop it (the model then uses its default). + if reasoning is None or not reasoning.omit_temperature: + message_kwargs["extra_body"] = {"temperature": iface.temperature} response = iface._anthropic_client.messages.create(**message_kwargs) diff --git a/agent_core/core/impl/llm/transports/bedrock_converse.py b/agent_core/core/impl/llm/transports/bedrock_converse.py index 202231c8..bbf3abe2 100644 --- a/agent_core/core/impl/llm/transports/bedrock_converse.py +++ b/agent_core/core/impl/llm/transports/bedrock_converse.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error from agent_core.utils.logger import logger @@ -70,6 +71,10 @@ def generate( else [{"role": "user", "content": [{"text": user_prompt}]}] ) + # Reasoning default for this exact model (None: no rule, so the + # request is shaped exactly as it was before reasoning defaults). + reasoning = iface.reasoning_decision() + converse_kwargs: Dict[str, Any] = { "modelId": iface.model, "messages": converse_messages, @@ -78,6 +83,19 @@ def generate( "maxTokens": iface.max_tokens, }, } + if reasoning is not None: + # Claude on Converse takes the Messages-API thinking/effort keys + # through additionalModelRequestFields. Thinking tokens count + # against maxTokens, and thinking is incompatible with (4.5/4.6) + # or rejects (4.7+) a non-default temperature. + converse_kwargs["additionalModelRequestFields"] = ( + reasoning_wire.anthropic_fields(reasoning) + ) + converse_kwargs["inferenceConfig"]["maxTokens"] = reasoning.output_cap( + iface.max_tokens + ) + if reasoning.omit_temperature: + del converse_kwargs["inferenceConfig"]["temperature"] if system_prompt: # When messages already carry a cachePoint (multi-turn first diff --git a/agent_core/core/impl/llm/transports/chat_completions.py b/agent_core/core/impl/llm/transports/chat_completions.py index 1420b0ce..f0871153 100644 --- a/agent_core/core/impl/llm/transports/chat_completions.py +++ b/agent_core/core/impl/llm/transports/chat_completions.py @@ -19,6 +19,7 @@ import requests from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error, provider_display_name from agent_core.core.models.registry import ( @@ -114,9 +115,14 @@ def generate_openai( "model": iface.model, "messages": messages, } + # Reasoning default for this exact model (None: no rule, so the + # request is shaped exactly as it was before reasoning defaults). + reasoning = iface.reasoning_decision() _profile = _get_registry().get(iface.provider) _temp = _resolve_temperature(_profile, iface.temperature) - if _temp is not _OMIT_TEMPERATURE: + if _temp is not _OMIT_TEMPERATURE and not ( + reasoning is not None and reasoning.omit_temperature + ): request_kwargs["temperature"] = _temp # Output tokens: cap the VALUE to the provider's output limit (several @@ -124,7 +130,13 @@ def generate_openai( # when it's exceeded), and pick the FIELD NAME per provider policy # (profile.uses_max_completion_tokens: OpenAI/Cerebras/MiniMax/Groq # take 'max_completion_tokens'; everyone else legacy 'max_tokens'). - _max_tokens_value = iface.max_tokens + # Reasoning tokens count against this cap, so a reasoning default + # raises it (never lowers it) to leave room for the answer. + _max_tokens_value = ( + iface.max_tokens + if reasoning is None + else reasoning.output_cap(iface.max_tokens) + ) if _profile is not None and _profile.max_output_tokens: _max_tokens_value = min(_max_tokens_value, _profile.max_output_tokens) uses_max_completion_tokens = ( @@ -199,6 +211,13 @@ def generate_openai( f"[OPENROUTER] Anthropic cache_control: {cache_control} (model={iface.model})" ) + if reasoning is not None: + reasoning_top, reasoning_extra = reasoning_wire.chat_completions_fields( + reasoning + ) + request_kwargs.update(reasoning_top) + extra_body.update(reasoning_extra) + if extra_body: request_kwargs["extra_body"] = extra_body diff --git a/agent_core/core/impl/llm/transports/gemini_native.py b/agent_core/core/impl/llm/transports/gemini_native.py index 0254f738..4ddc8305 100644 --- a/agent_core/core/impl/llm/transports/gemini_native.py +++ b/agent_core/core/impl/llm/transports/gemini_native.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error from agent_core.utils.logger import logger @@ -52,11 +53,24 @@ def generate( # Per-call reasoning cap, set by callers that pass thinking_budget (e.g. the # entity-judge pipeline). Rides the shared per-call context so no transport - # signature changes; None for every ordinary call, in which case Gemini's - # default thinking behaviour is unchanged. + # signature changes. An explicit per-call budget wins over the model's + # reasoning default; otherwise the default for this exact model applies + # (None: no rule, so Gemini's own default thinking is left untouched). from agent_core.core.impl.llm.interface import _llm_call_ctx - thinking_budget = (_llm_call_ctx.get() or {}).get("thinking_budget") + caller_budget = (_llm_call_ctx.get() or {}).get("thinking_budget") + reasoning = iface.reasoning_decision() if caller_budget is None else None + if caller_budget is not None: + thinking: Dict[str, Any] = {"thinking_budget": caller_budget} + elif reasoning is not None: + thinking = reasoning_wire.gemini_thinking_kwargs(reasoning) + else: + thinking = {} + # Thinking tokens count against maxOutputTokens, so a reasoning default + # raises the cap (never lowers it) to leave room for the answer. + max_output_tokens = ( + iface.max_tokens if reasoning is None else reasoning.output_cap(iface.max_tokens) + ) token_count_input = token_count_output = 0 cached_tokens = 0 @@ -85,8 +99,9 @@ def generate( contents=contents_override, system_prompt=system_prompt, temperature=iface.temperature, - max_output_tokens=iface.max_tokens, + max_output_tokens=max_output_tokens, json_mode=json_mode, + **thinking, ) else: # Use explicit caching when: @@ -116,7 +131,8 @@ def generate( user_prompt=user_prompt, call_type=call_type, temperature=iface.temperature, - max_tokens=iface.max_tokens, + max_tokens=max_output_tokens, + **thinking, ) else: # Fall back to implicit caching (or no caching for short prompts) @@ -125,9 +141,9 @@ def generate( prompt=user_prompt, system_prompt=system_prompt, temperature=iface.temperature, - max_output_tokens=iface.max_tokens, + max_output_tokens=max_output_tokens, json_mode=json_mode, - thinking_budget=thinking_budget, + **thinking, ) # Extract response data diff --git a/agent_core/core/llm/google_gemini_client.py b/agent_core/core/llm/google_gemini_client.py index db9f6290..98e2593b 100644 --- a/agent_core/core/llm/google_gemini_client.py +++ b/agent_core/core/llm/google_gemini_client.py @@ -38,6 +38,25 @@ def _normalise_model_name(model: str) -> str: return model if model.startswith("models/") else f"models/{model}" +def _thinking_config( + thinking_budget: Optional[int], thinking_level: Optional[str] +) -> Optional[Dict[str, Any]]: + """Build ``generationConfig.thinkingConfig``, or None to omit it. + + ``thinkingBudget`` is the Gemini 2.5 knob and ``thinkingLevel`` the + Gemini 3.x one; the API rejects a request that sets both with a 400. + """ + if thinking_budget is not None and thinking_level is not None: + raise ValueError( + "Gemini accepts either thinking_budget or thinking_level, not both." + ) + if thinking_budget is not None: + return {"thinkingBudget": thinking_budget} + if thinking_level is not None: + return {"thinkingLevel": thinking_level} + return None + + class GeminiClient: """Lightweight REST client for Gemini models. @@ -94,6 +113,7 @@ def generate_text( max_output_tokens: Optional[int] = None, json_mode: bool = False, thinking_budget: Optional[int] = None, + thinking_level: Optional[str] = None, ) -> Dict[str, Any]: """Generate text for a purely textual prompt. @@ -121,6 +141,9 @@ def generate_text( (and can exhaust maxOutputTokens on thoughts alone, emitting no text — finishReason=MAX_TOKENS, parts_count=0). Set it to reserve output room for the actual answer. + thinking_level: Optional reasoning level for Gemini 3.x thinking + models (``thinkingLevel``). Mutually exclusive with + ``thinking_budget``. Returns: Dict with generation results and token counts @@ -139,8 +162,9 @@ def generate_text( generation_config["maxOutputTokens"] = max_output_tokens if json_mode: generation_config["responseMimeType"] = "application/json" - if thinking_budget is not None: - generation_config["thinkingConfig"] = {"thinkingBudget": thinking_budget} + thinking_config = _thinking_config(thinking_budget, thinking_level) + if thinking_config is not None: + generation_config["thinkingConfig"] = thinking_config payload: Dict[str, Any] = {"contents": contents} if system_prompt: @@ -181,6 +205,8 @@ def generate_text_multiturn( temperature: Optional[float] = None, max_output_tokens: Optional[int] = None, json_mode: bool = False, + thinking_budget: Optional[int] = None, + thinking_level: Optional[str] = None, ) -> Dict[str, Any]: """Generate text from a pre-built multi-turn `contents` array. @@ -199,6 +225,10 @@ def generate_text_multiturn( temperature: Sampling temperature. max_output_tokens: Output token cap. json_mode: Force JSON response. + thinking_budget: Reasoning token budget (Gemini 2.5), see + ``generate_text``. + thinking_level: Reasoning level (Gemini 3.x), see + ``generate_text``. Returns: Same shape as ``generate_text``. @@ -210,6 +240,9 @@ def generate_text_multiturn( generation_config["maxOutputTokens"] = max_output_tokens if json_mode: generation_config["responseMimeType"] = "application/json" + thinking_config = _thinking_config(thinking_budget, thinking_level) + if thinking_config is not None: + generation_config["thinkingConfig"] = thinking_config payload: Dict[str, Any] = {"contents": contents} if system_prompt: @@ -425,6 +458,8 @@ def generate_text_with_cache( temperature: Optional[float] = None, max_output_tokens: Optional[int] = None, json_mode: bool = False, + thinking_budget: Optional[int] = None, + thinking_level: Optional[str] = None, ) -> Dict[str, Any]: """Generate text using an explicit cache. @@ -438,6 +473,10 @@ def generate_text_with_cache( temperature: Sampling temperature max_output_tokens: Maximum output tokens json_mode: If True, enforce JSON output format + thinking_budget: Reasoning token budget (Gemini 2.5), see + ``generate_text``. + thinking_level: Reasoning level (Gemini 3.x), see + ``generate_text``. Returns: Dict with tokens_used, content, prompt_tokens, completion_tokens, cached_tokens @@ -456,6 +495,9 @@ def generate_text_with_cache( generation_config["maxOutputTokens"] = max_output_tokens if json_mode: generation_config["responseMimeType"] = "application/json" + thinking_config = _thinking_config(thinking_budget, thinking_level) + if thinking_config is not None: + generation_config["thinkingConfig"] = thinking_config payload: Dict[str, Any] = { "contents": contents, diff --git a/agent_core/core/models/chatgpt_subscription_client.py b/agent_core/core/models/chatgpt_subscription_client.py index d29c9493..c2e82fef 100644 --- a/agent_core/core/models/chatgpt_subscription_client.py +++ b/agent_core/core/models/chatgpt_subscription_client.py @@ -15,8 +15,10 @@ - ``store: false`` ("Store must be set to false") - ``stream: true`` ("Stream must be set to true"); aggregated below -- ``reasoning.effort: `` ("none" for non-codex 5.1/5.2, - "low" for codex variants — "minimal" is rejected by the backend) +- ``reasoning.effort``: the caller's ``reasoning_effort`` (resolved per model + by agent_core/core/models/reasoning.py under the ``openai_subscription`` + surface), or ``"medium"`` for a model without a rule. The block itself is + required, so it is never omitted. - ``reasoning.summary: "auto"`` - ``include: ["reasoning.encrypted_content"]`` (mandatory under ``store=false`` so the model can keep its own @@ -161,14 +163,23 @@ def __init__( # silently honoring "best-effort" semantics is fine for fields that just # don't apply at this backend (e.g. ``max_tokens`` becomes "let the # server decide" rather than a hard failure). -def _codex_reasoning_config(_model: str) -> Dict[str, str]: +#: Effort for a model the reasoning table has no row for. The Codex backend +#: requires a ``reasoning`` block on every request, so unlike every other +#: provider the parameter cannot simply be left out; "medium" is the Codex +#: CLI default and what this translator always sent before per-model rules. +_CODEX_UNRULED_EFFORT = "medium" + + +def _codex_reasoning_config(requested_effort: Any) -> Dict[str, str]: """Build the ``reasoning`` block Codex requires on every request. - "medium" effort matches the Codex CLI default — fast enough for - JSON action-decision loops, deliberate enough that the model - follows instruction-following. ``"auto"`` summary matches the CLI. + ``requested_effort`` is the caller's Chat-Completions + ``reasoning_effort``: the per-model default the chat_completions + transport resolved, or None when the model has no rule. + ``"auto"`` summary matches the Codex CLI. """ - return {"effort": "medium", "summary": "auto"} + effort = requested_effort if requested_effort else _CODEX_UNRULED_EFFORT + return {"effort": str(effort), "summary": "auto"} def _extract_instructions( @@ -231,7 +242,7 @@ def _translate_request( out["input"] = _normalize_messages(rest) # ``reasoning`` is REQUIRED for every gpt-5.x model on Codex. - out["reasoning"] = _codex_reasoning_config(model) + out["reasoning"] = _codex_reasoning_config(kwargs.get("reasoning_effort")) # ``text.verbosity`` is required by the Codex backend to know how # long a response to produce. The reference impl always sets it; diff --git a/agent_core/core/models/factory.py b/agent_core/core/models/factory.py index 2006e5d8..e360c96b 100644 --- a/agent_core/core/models/factory.py +++ b/agent_core/core/models/factory.py @@ -11,8 +11,10 @@ try: import boto3 # type: ignore[import] + from botocore.config import Config as _BotocoreConfig # type: ignore[import] except ImportError: # pragma: no cover — boto3 is an optional extra boto3 = None # type: ignore[assignment] + _BotocoreConfig = None # type: ignore[assignment] from agent_core.core.models.types import InterfaceType from agent_core.core.models.provider_config import PROVIDER_CONFIG @@ -24,6 +26,12 @@ logger = logging.getLogger(__name__) +# Read timeout for Bedrock Converse calls. botocore's default is 60 seconds, +# which a Claude response with thinking can exceed; AWS documents a +# 60-minute server-side limit for Claude 4 and later. 600 seconds matches the +# Anthropic SDK's default request timeout used by the direct transport. +_BEDROCK_READ_TIMEOUT_SECONDS = 600 + # Derived from provider profiles (Phase 1, docs/PROVIDER_LAYER_CATCHUP.md). # OpenRouter proxy routing exists because some direct APIs are geo-restricted # for most international users; the per-provider data lives on the profiles. @@ -618,7 +626,12 @@ def create( ) try: - client_kwargs = {"region_name": region} + client_kwargs = { + "region_name": region, + "config": _BotocoreConfig( + read_timeout=_BEDROCK_READ_TIMEOUT_SECONDS + ), + } if access_key and secret_key: client_kwargs["aws_access_key_id"] = access_key client_kwargs["aws_secret_access_key"] = secret_key diff --git a/agent_core/core/models/reasoning.py b/agent_core/core/models/reasoning.py new file mode 100644 index 00000000..3f2301ce --- /dev/null +++ b/agent_core/core/models/reasoning.py @@ -0,0 +1,814 @@ +# -*- coding: utf-8 -*- +"""Default reasoning effort ("thinking") per model. + +Why this exists +--------------- +Reasoning models need an explicit request parameter to think at a useful +depth, and several of them do no reasoning at all when it is omitted +(OpenAI gpt-5.1 through gpt-5.4 default to ``none``; Claude Sonnet 4.6 and +Opus 4.6-4.8 run without thinking unless it is requested). The parameter is +also model-specific: its name, shape, and accepted values differ between +models of the SAME provider, and a value a model does not accept is a hard +HTTP 400 on the first request. + +So defaults live in a hard-coded table keyed by provider and EXACT model id +(no prefix or substring matching). A model that is not in the table gets no +reasoning parameter at all, and its request stays byte-identical to the one +sent before this module existed (pinned by the ``unruled_*`` golden payload +snapshots under tests/llm/golden/). + +Choosing the level +------------------ +``ReasoningRule.levels`` lists the levels that turn reasoning on, weakest +first. "Off" values (``none``, ``disabled``, ``minimal``) and values a +provider silently maps onto another level are left out, so the tuple holds +only levels that behave differently. The default level is: + +1. one level below the model's strongest level (``LEVELS_BELOW_MAX``); +2. never stronger than ``LEVEL_CEILING`` (``high``); +3. but if that is weaker than the level the provider already applies when + the parameter is omitted (``provider_default``), the model's strongest + level is used instead, so a default is never lowered. + +Token-budget models (Claude 4.5, Gemini 2.5) have no named levels; their +rows name four budget rungs ``BUDGET_RUNGS`` so the same rule picks the +third rung ("high"). + +Adding a model +-------------- +Add its exact id (every alias and dated snapshot the provider accepts) to a +rule below, with the ordered levels from the provider's documentation and +the documentation URL in ``source``. The unit tests in +tests/llm/test_reasoning_rules.py validate every row. + +The keys follow the ``/`` shape of the model catalog +planned in docs/PROVIDER_LAYER_CATCHUP.md section 10, so the rows can move +into that catalog as a column without a rewrite. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from enum import Enum +from typing import Dict, Mapping, Optional, Tuple + +# ─────────────────────────────── policy ─────────────────────────────── + +#: How many levels below the model's strongest level the default sits. +LEVELS_BELOW_MAX = 1 + +#: The strongest level ever chosen as a default (unless the provider's own +#: default is already stronger; see rule 3 in the module docstring). +LEVEL_CEILING = "high" + +#: Every level name any provider uses, weakest first. Rows must use these. +KNOWN_LEVELS: Tuple[str, ...] = ("minimal", "low", "medium", "high", "xhigh", "max") + +#: Rung names for token-budget models, weakest first. +BUDGET_RUNGS: Tuple[str, ...] = ("low", "medium", "high", "max") + +#: The auth mode whose OpenAI requests go to the ChatGPT-subscription Codex +#: backend, which serves a different model catalogue with different levels. +SUBSCRIPTION_AUTH_MODE = "subscription" + + +class ReasoningWire(str, Enum): + """How a reasoning default is expressed in a request.""" + + #: Top-level ``reasoning_effort: `` on a Chat Completions request. + #: The ChatGPT-subscription translator maps it to ``reasoning.effort``. + EFFORT = "effort" + #: Anthropic ``thinking: {"type": "adaptive"}`` plus + #: ``output_config: {"effort": }`` (Claude 4.6 and later). + ANTHROPIC_ADAPTIVE = "anthropic_adaptive" + #: Anthropic ``thinking: {"type": "enabled", "budget_tokens": }`` + #: (Claude 4.5 generation). + ANTHROPIC_BUDGET = "anthropic_budget" + #: Gemini ``thinkingConfig: {"thinkingLevel": }`` (Gemini 3.x). + GEMINI_LEVEL = "gemini_level" + #: Gemini ``thinkingConfig: {"thinkingBudget": }`` (Gemini 2.5). + GEMINI_BUDGET = "gemini_budget" + #: OpenRouter ``reasoning: {"effort": }``. + OPENROUTER_EFFORT = "openrouter_effort" + #: OpenRouter ``reasoning: {"max_tokens": }``. + OPENROUTER_BUDGET = "openrouter_budget" + + +#: Wires whose rows carry token budgets instead of named levels. +BUDGET_WIRES = frozenset( + { + ReasoningWire.ANTHROPIC_BUDGET, + ReasoningWire.GEMINI_BUDGET, + ReasoningWire.OPENROUTER_BUDGET, + } +) + + +@dataclass(frozen=True) +class ReasoningRule: + """Reasoning capabilities of one model (or of models that share them).""" + + #: How the default is expressed in the request. + wire: ReasoningWire + #: Levels that turn reasoning on, weakest first (``BUDGET_RUNGS`` for + #: budget wires). + levels: Tuple[str, ...] + #: Level the provider applies when the parameter is omitted, when that + #: is one of ``levels``. None when omitting it means no reasoning, a + #: provider-chosen dynamic amount, or an "off" value. + provider_default: Optional[str] + #: Documentation the row was taken from. + source: str + #: Budget-wire rows: token budget per rung, aligned with ``levels``. + budgets: Tuple[int, ...] = () + #: Output-token cap to request while reasoning is on. Reasoning tokens + #: count against the provider's output cap, so it must leave room for + #: the answer. 0 keeps the caller's cap unchanged. + output_tokens: int = 0 + #: The model rejects ``temperature`` while reasoning is on. + omit_temperature: bool = False + + +@dataclass(frozen=True) +class ReasoningDecision: + """The reasoning default resolved for one (provider, model) pair.""" + + #: ``/`` of the row that produced this decision. + key: str + wire: ReasoningWire + level: str + #: Token budget for budget wires, else None. + budget_tokens: Optional[int] + output_tokens: int + omit_temperature: bool + + def output_cap(self, base: int) -> int: + """Output-token cap for a request whose cap is otherwise ``base``. + + Never lowers ``base``: reasoning only ever adds room. + """ + return max(base, self.output_tokens) + + def describe(self) -> str: + """One-line human-readable summary for logs.""" + if self.budget_tokens is not None: + return f"{self.wire.value} budget={self.budget_tokens} ({self.level})" + return f"{self.wire.value} level={self.level}" + + +def target_level(rule: ReasoningRule) -> str: + """Pick the default level for ``rule`` (see the module docstring).""" + levels = rule.levels + index = max(len(levels) - 1 - LEVELS_BELOW_MAX, 0) + if LEVEL_CEILING in levels: + index = min(index, levels.index(LEVEL_CEILING)) + if rule.provider_default in levels and index < levels.index(rule.provider_default): + index = len(levels) - 1 + return levels[index] + + +def reasoning_surface(provider: str, auth_mode: str) -> str: + """Table surface for a provider and auth mode. + + OpenAI requests made with a ChatGPT-subscription login go to the Codex + backend, whose model catalogue and levels differ from the public API. + """ + if provider == "openai" and auth_mode == SUBSCRIPTION_AUTH_MODE: + return "openai_subscription" + return provider + + +def resolve_reasoning( + provider: Optional[str], + model: Optional[str], + auth_mode: str = "api_key", +) -> Optional[ReasoningDecision]: + """Resolve the reasoning default for a model, or None if it has no row.""" + if not provider or not model: + return None + surface = reasoning_surface(provider, auth_mode) + rule = REASONING_RULES.get(surface, {}).get(model) + if rule is None: + return None + level = target_level(rule) + budget = ( + rule.budgets[rule.levels.index(level)] if rule.wire in BUDGET_WIRES else None + ) + return ReasoningDecision( + key=f"{surface}/{model}", + wire=rule.wire, + level=level, + budget_tokens=budget, + output_tokens=rule.output_tokens, + omit_temperature=rule.omit_temperature, + ) + + +def _models(rule: ReasoningRule, *model_ids: str) -> Dict[str, ReasoningRule]: + return {model_id: rule for model_id in model_ids} + + +def _surface(*groups: Dict[str, ReasoningRule]) -> Dict[str, ReasoningRule]: + """Merge one provider's ``_models`` groups, refusing duplicate ids. + + A plain dict merge would let a second listing of an id silently replace + the first, sending a value its real model may reject. + """ + merged: Dict[str, ReasoningRule] = {} + for group in groups: + for model_id, rule in group.items(): + if model_id in merged: + raise ValueError( + f"Model id {model_id!r} is listed twice in the reasoning table." + ) + merged[model_id] = rule + return merged + + +def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: + """A Bedrock base model id plus its documented inference-profile ids. + + ``profile_prefixes`` are the geo/global prefixes the model's AWS model + card lists (for example ``us`` gives ``us.``); each resulting + id is matched exactly like any other row. + """ + return (base_id, *(f"{prefix}.{base_id}" for prefix in profile_prefixes)) + + +# ─────────────────────────────── rules ──────────────────────────────── +# +# Output caps. Non-streaming Anthropic requests are refused by the Anthropic +# SDK above 21,333 max_tokens (anthropic/_base_client.py +# _calculate_nonstreaming_timeout), so Anthropic-family rows stay just below +# it. OpenAI-style effort rows use 32,000: OpenAI recommends reserving at +# least 25,000 tokens for reasoning plus output +# (https://developers.openai.com/api/docs/guides/reasoning). + +_ANTHROPIC_OUTPUT_TOKENS = 21_000 +_EFFORT_OUTPUT_TOKENS = 32_000 + +# ── OpenAI public API (Chat Completions ``reasoning_effort``) ── +# Values and defaults: https://developers.openai.com/api/docs/models/. +# Models before gpt-5.1 reject "none"; gpt-5.1+ reject "minimal"; xhigh +# arrived with gpt-5.2 and max with gpt-5.6. Temperature is rejected whenever +# effort is not "none" (the OpenAI profile already omits it). + +_OPENAI_DOCS = "https://developers.openai.com/api/docs/models" + +_OPENAI_GPT5 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high"), + provider_default="medium", + source=f"{_OPENAI_DOCS}/gpt-5", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENAI_O_SERIES = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high"), + provider_default="medium", + source="https://developers.openai.com/api/docs/guides/reasoning", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENAI_GPT51 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high"), + provider_default=None, # "none": no reasoning + source=f"{_OPENAI_DOCS}/gpt-5.1", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENAI_GPT52_TO_54 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default=None, # "none": no reasoning + source=f"{_OPENAI_DOCS}/gpt-5.2", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENAI_GPT55 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default="medium", + source=f"{_OPENAI_DOCS}/gpt-5.5", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENAI_GPT56_PLUS = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="medium", + source=f"{_OPENAI_DOCS}/gpt-6-sol", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +#: GPT-6 Astra rejects "none" and documents no default. +_OPENAI_GPT6_ASTRA = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default=None, + source=f"{_OPENAI_DOCS}/gpt-6-astra", + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) + +# ── ChatGPT subscription (Codex backend ``reasoning.effort``) ── +# The translator always sends a reasoning block and drops output caps, so +# output_tokens stays 0. Levels are the catalogue's supported_reasoning_levels +# minus the client-side "ultra" alias +# (https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json). + +_CODEX_CATALOG = ( + "https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json" +) +_CODEX_GPT55 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default="medium", + source=_CODEX_CATALOG, +) +_CODEX_MAX_LADDER_LOW_DEFAULT = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="low", + source=_CODEX_CATALOG, +) +_CODEX_MAX_LADDER_MEDIUM_DEFAULT = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="medium", + source=_CODEX_CATALOG, +) + +# ── OpenRouter (``reasoning`` object) ── +# Levels: live GET https://openrouter.ai/api/v1/models ``reasoning`` +# (supported_efforts). Claude 4.5 slugs expose no effort selector, only a +# token budget, which must be >= 1024 and strictly below the request +# max_tokens (https://openrouter.ai/docs/guides/best-practices/reasoning-tokens). +# Explicitly sent sampling parameters are forwarded upstream, where thinking +# rejects them, so every OpenRouter row drops temperature. + +_OPENROUTER_MODELS = "https://openrouter.ai/api/v1/models" +_OPENROUTER_CLAUDE_BUDGET = ReasoningRule( + wire=ReasoningWire.OPENROUTER_BUDGET, + levels=BUDGET_RUNGS, + provider_default=None, + source="https://openrouter.ai/docs/guides/best-practices/reasoning-tokens", + budgets=(4_096, 8_192, 12_288, 16_384), + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENROUTER_CLAUDE_SONNET_46 = ReasoningRule( + wire=ReasoningWire.OPENROUTER_EFFORT, + levels=("low", "medium", "high", "max"), + provider_default="medium", + source=_OPENROUTER_MODELS, + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENROUTER_CLAUDE_OPUS_46 = replace( + _OPENROUTER_CLAUDE_SONNET_46, provider_default="high" +) +_OPENROUTER_OPENAI_GPT5 = ReasoningRule( + wire=ReasoningWire.OPENROUTER_EFFORT, + levels=("low", "medium", "high"), + provider_default="medium", + source=_OPENROUTER_MODELS, + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENROUTER_OPENAI_GPT51 = replace(_OPENROUTER_OPENAI_GPT5, provider_default=None) +_OPENROUTER_OPENAI_XHIGH = ReasoningRule( + wire=ReasoningWire.OPENROUTER_EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default="medium", + source=_OPENROUTER_MODELS, + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) +_OPENROUTER_OPENAI_MAX = ReasoningRule( + wire=ReasoningWire.OPENROUTER_EFFORT, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="medium", + source=_OPENROUTER_MODELS, + output_tokens=_EFFORT_OUTPUT_TOKENS, + omit_temperature=True, +) + +# ── Google Gemini (generateContent ``thinkingConfig``) ── +# https://ai.google.dev/gemini-api/docs/generate-content/thinking. 2.5 models +# take a token budget only (thinkingLevel is a 400 on them); 3.x models take +# a lowercase thinkingLevel. Thinking tokens count toward maxOutputTokens +# (65,536 on every model below), so the cap leaves 8,192 tokens of answer on +# top of the chosen budget. "minimal" is excluded: it is near-off, and an +# error on 3.1-pro / 3.7-flash / 3.8-flash. + +_GEMINI_THINKING_DOCS = ( + "https://ai.google.dev/gemini-api/docs/generate-content/thinking" +) +_GEMINI_OUTPUT_TOKENS = 32_768 +_GEMINI_25_PRO = ReasoningRule( + wire=ReasoningWire.GEMINI_BUDGET, + levels=BUDGET_RUNGS, + provider_default=None, # dynamic thinking + source=_GEMINI_THINKING_DOCS, + budgets=(8_192, 16_384, 24_576, 32_768), + output_tokens=24_576 + 8_192, +) +_GEMINI_25_FLASH = ReasoningRule( + wire=ReasoningWire.GEMINI_BUDGET, + levels=BUDGET_RUNGS, + provider_default=None, # dynamic (flash) or no thinking (flash-lite) + source=_GEMINI_THINKING_DOCS, + budgets=(6_144, 12_288, 18_432, 24_576), + output_tokens=18_432 + 8_192, +) +_GEMINI_3_HIGH_DEFAULT = ReasoningRule( + wire=ReasoningWire.GEMINI_LEVEL, + levels=("low", "medium", "high"), + provider_default="high", + source=_GEMINI_THINKING_DOCS, + output_tokens=_GEMINI_OUTPUT_TOKENS, +) +_GEMINI_3_MEDIUM_DEFAULT = replace(_GEMINI_3_HIGH_DEFAULT, provider_default="medium") +_GEMINI_3_MINIMAL_DEFAULT = replace(_GEMINI_3_HIGH_DEFAULT, provider_default=None) + +# ── xAI (Chat Completions ``reasoning_effort``) ── +# https://docs.x.ai/developers/models/. No Grok model accepts "max", and +# "none" is documented only for grok-4.3. grok-4.5 treats "xhigh" as "high", +# so its distinct levels stop at high. + +_XAI_DOCS = "https://docs.x.ai/developers/models" +_XAI_GROK_43 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default="low", + source=f"{_XAI_DOCS}/grok-4.3", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) +_XAI_GROK_45 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high"), + provider_default="high", + source="https://docs.x.ai/developers/model-capabilities/text/reasoning", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) +_XAI_GROK_46_PLUS = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high", "xhigh"), + provider_default="high", + source="https://docs.x.ai/developers/model-capabilities/text/reasoning", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) + +# ── DeepSeek (``reasoning_effort``) ── +# https://api-docs.deepseek.com/api/create-chat-completion: none/low/high/max +# (minimal maps to low, medium/xhigh to high), default high; max_tokens up to +# 393,216. Thinking mode ignores temperature without an error. + +_DEEPSEEK_V4 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "high", "max"), + provider_default="high", + source="https://api-docs.deepseek.com/api/create-chat-completion", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) + +# ── Z.ai GLM (``reasoning_effort``, standard route) ── +# https://docs.z.ai/api-reference/llm/chat-completion, default max, 128K +# output. glm-5.3 accepts only low/high/max; glm-5.2 maps low/medium to high +# and xhigh to max, so its distinct levels are high and max. + +_GLM_DOCS = "https://docs.z.ai/api-reference/llm/chat-completion" +_GLM_53 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "high", "max"), + provider_default="max", + source=_GLM_DOCS, + output_tokens=_EFFORT_OUTPUT_TOKENS, +) +_GLM_52 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("high", "max"), + provider_default="max", + source=_GLM_DOCS, + output_tokens=_EFFORT_OUTPUT_TOKENS, +) + +# ── Moonshot Kimi (``reasoning_effort``) ── +# https://platform.kimi.ai/docs/guide/kimi-k3-quickstart: low/high/max, +# default max; always thinks. Temperature is fixed (the profile omits it). + +_KIMI_K3 = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "high", "max"), + provider_default="max", + source="https://platform.kimi.ai/docs/guide/kimi-k3-quickstart", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) + +# ── gpt-oss on Groq / Cerebras (``reasoning_effort``) ── +# low/medium/high, default medium; out-of-set values are a 400. Output caps +# are clamped further by each provider profile's max_output_tokens. + +_GPT_OSS_GROQ = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=("low", "medium", "high"), + provider_default="medium", + source="https://console.groq.com/docs/api-reference", + output_tokens=_EFFORT_OUTPUT_TOKENS, +) +_GPT_OSS_CEREBRAS = replace( + _GPT_OSS_GROQ, source="https://inference-docs.cerebras.ai/capabilities/reasoning" +) + +_CLAUDE_DOCS = "https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking" +_CLAUDE_BUDGET_DOCS = ( + "https://platform.claude.com/docs/en/build-with-claude/extended-thinking" +) + +#: Claude 4.6: adaptive thinking is off unless requested; effort has no +#: xhigh; sampling parameters are incompatible with thinking. +_CLAUDE_46 = ReasoningRule( + wire=ReasoningWire.ANTHROPIC_ADAPTIVE, + levels=("low", "medium", "high", "max"), + provider_default="high", + source=_CLAUDE_DOCS, + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) + +#: Claude 4.7 and later: adaptive is the only thinking mode, the full effort +#: ladder exists, and non-default sampling parameters are a 400. +_CLAUDE_47_PLUS = ReasoningRule( + wire=ReasoningWire.ANTHROPIC_ADAPTIVE, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="high", + source=_CLAUDE_DOCS, + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) + +#: Claude Opus 5.5: same surface, but its effort default is medium. +_CLAUDE_OPUS_55 = ReasoningRule( + wire=ReasoningWire.ANTHROPIC_ADAPTIVE, + levels=("low", "medium", "high", "xhigh", "max"), + provider_default="medium", + source=_CLAUDE_DOCS, + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) + +#: Claude 4.5 generation: manual budget thinking only (adaptive is a 400, +#: and effort is a 400 on Sonnet 4.5 / Haiku 4.5, so it is never sent). +#: budget_tokens must be >= 1024 and < max_tokens. +_CLAUDE_45_BUDGET = ReasoningRule( + wire=ReasoningWire.ANTHROPIC_BUDGET, + levels=BUDGET_RUNGS, + provider_default=None, + source=_CLAUDE_BUDGET_DOCS, + budgets=(4_096, 8_192, 12_288, 16_384), + output_tokens=_ANTHROPIC_OUTPUT_TOKENS, + omit_temperature=True, +) + + +#: Claude on AWS Bedrock Converse takes the same thinking/effort keys through +#: additionalModelRequestFields, with the same per-model rules +#: (https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-adaptive-thinking.html, +#: .../claude-messages-extended-thinking.html). Ids and inference-profile +#: prefixes come from each model's AWS model card. +_BEDROCK_DOCS = ( + "https://docs.aws.amazon.com/bedrock/latest/userguide/" + "claude-messages-adaptive-thinking.html" +) +_BEDROCK_CLAUDE_46 = replace(_CLAUDE_46, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_47_PLUS = replace(_CLAUDE_47_PLUS, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_OPUS_55 = replace(_CLAUDE_OPUS_55, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_45_BUDGET = replace( + _CLAUDE_45_BUDGET, + source=( + "https://docs.aws.amazon.com/bedrock/latest/userguide/" + "claude-messages-extended-thinking.html" + ), +) + + +REASONING_RULES: Mapping[str, Mapping[str, ReasoningRule]] = { + "openai": _surface( + _models( + _OPENAI_GPT5, + "gpt-5", + "gpt-5-2025-08-07", + "gpt-5-mini", + "gpt-5-mini-2025-08-07", + "gpt-5-nano", + "gpt-5-nano-2025-08-07", + ), + _models( + _OPENAI_O_SERIES, + "o3", + "o3-2025-04-16", + "o3-mini", + "o3-mini-2025-01-31", + "o4-mini", + "o4-mini-2025-04-16", + ), + _models(_OPENAI_GPT51, "gpt-5.1", "gpt-5.1-2025-11-13"), + _models( + _OPENAI_GPT52_TO_54, + "gpt-5.2", + "gpt-5.2-2025-12-11", + "gpt-5.4", + "gpt-5.4-2026-03-05", + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", + ), + _models(_OPENAI_GPT55, "gpt-5.5", "gpt-5.5-2026-04-23"), + _models( + _OPENAI_GPT56_PLUS, + "gpt-5.6", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + ), + _models(_OPENAI_GPT6_ASTRA, "gpt-6-astra"), + ), + "openai_subscription": _surface( + _models(_CODEX_GPT55, "gpt-5.5"), + _models( + _CODEX_MAX_LADDER_LOW_DEFAULT, "gpt-5.6-sol", "gpt-6-astra", "gpt-6.1-sol" + ), + _models( + _CODEX_MAX_LADDER_MEDIUM_DEFAULT, + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-sol", + "gpt-6-luna", + ), + ), + "openrouter": _surface( + _models( + _OPENROUTER_CLAUDE_BUDGET, + "anthropic/claude-sonnet-4.5", + "anthropic/claude-haiku-4.5", + "anthropic/claude-opus-4.5", + ), + _models(_OPENROUTER_CLAUDE_SONNET_46, "anthropic/claude-sonnet-4.6"), + _models(_OPENROUTER_CLAUDE_OPUS_46, "anthropic/claude-opus-4.6"), + _models(_OPENROUTER_OPENAI_GPT5, "openai/gpt-5", "openai/gpt-5-mini"), + _models(_OPENROUTER_OPENAI_GPT51, "openai/gpt-5.1"), + _models( + _OPENROUTER_OPENAI_XHIGH, + "openai/gpt-5.2", + "openai/gpt-5.4", + "openai/gpt-5.5", + ), + _models( + _OPENROUTER_OPENAI_MAX, + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna", + "openai/gpt-6-sol", + "openai/gpt-6-luna", + "openai/gpt-6-astra", + "openai/gpt-6.1-sol", + ), + ), + "gemini": _surface( + _models(_GEMINI_25_PRO, "gemini-2.5-pro"), + _models(_GEMINI_25_FLASH, "gemini-2.5-flash", "gemini-2.5-flash-lite"), + _models( + _GEMINI_3_HIGH_DEFAULT, + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + ), + _models( + _GEMINI_3_MEDIUM_DEFAULT, + "gemini-3.5-flash", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + ), + _models( + _GEMINI_3_MINIMAL_DEFAULT, "gemini-3.1-flash-lite", "gemini-3.5-flash-lite" + ), + ), + "grok": _surface( + _models(_XAI_GROK_43, "grok-4.3", "grok-4.3-latest"), + _models(_XAI_GROK_45, "grok-4.5", "grok-4.5-latest"), + _models(_XAI_GROK_46_PLUS, "grok-4.6", "grok-4.7"), + ), + "deepseek": _surface( + _models( + _DEEPSEEK_V4, + "deepseek-flash", + "deepseek-v4-pro", + "deepseek-v4-flash", + "deepseek-v4-flash-vision-exp", + ), + ), + "glm": _surface( + _models(_GLM_53, "glm-5.3", "glm-5.3-flash"), + _models(_GLM_52, "glm-5.2"), + ), + "moonshot": _surface( + _models(_KIMI_K3, "kimi-k3"), + ), + "groq": _surface( + _models( + _GPT_OSS_GROQ, + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "openai/gpt-oss-safeguard-20b", + ), + ), + "cerebras": _surface( + _models(_GPT_OSS_CEREBRAS, "gpt-oss-120b"), + ), + "anthropic": _surface( + _models(_CLAUDE_46, "claude-sonnet-4-6", "claude-opus-4-6"), + _models( + _CLAUDE_47_PLUS, + "claude-opus-4-7", + "claude-opus-4-8", + "claude-sonnet-5", + "claude-sonnet-5-5", + "claude-opus-5", + "claude-fable-5", + "claude-fable-5-1", + "claude-mythos-5", + "claude-mythos-5-1", + ), + _models(_CLAUDE_OPUS_55, "claude-opus-5-5"), + _models( + _CLAUDE_45_BUDGET, + "claude-haiku-4-5", + "claude-haiku-4-5-20251001", + "claude-sonnet-4-5", + "claude-sonnet-4-5-20250929", + "claude-opus-4-5", + "claude-opus-4-5-20251101", + ), + ), + "bedrock": _surface( + _models( + _BEDROCK_CLAUDE_45_BUDGET, + *_bedrock_ids( + "anthropic.claude-haiku-4-5-20251001-v1:0", + "us", + "eu", + "au", + "jp", + "in", + "global", + ), + *_bedrock_ids( + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "us", + "eu", + "au", + "jp", + "global", + ), + *_bedrock_ids( + "anthropic.claude-opus-4-5-20251101-v1:0", "us", "eu", "global" + ), + ), + _models( + _BEDROCK_CLAUDE_46, + *_bedrock_ids( + "anthropic.claude-sonnet-4-6", "us", "eu", "au", "jp", "global" + ), + *_bedrock_ids("anthropic.claude-opus-4-6-v1", "us", "eu", "au", "global"), + ), + _models( + _BEDROCK_CLAUDE_47_PLUS, + *_bedrock_ids( + "anthropic.claude-opus-4-7", "us", "eu", "jp", "au", "global" + ), + *_bedrock_ids( + "anthropic.claude-opus-4-8", "us", "eu", "jp", "au", "global" + ), + *_bedrock_ids( + "anthropic.claude-sonnet-5", "us", "eu", "au", "in", "global" + ), + *_bedrock_ids("anthropic.claude-opus-5", "us", "eu", "au", "in", "global"), + *_bedrock_ids("anthropic.claude-fable-5", "us", "global"), + *_bedrock_ids("anthropic.claude-fable-5-1", "us", "global"), + ), + _models( + _BEDROCK_CLAUDE_OPUS_55, + *_bedrock_ids( + "anthropic.claude-opus-5-5", "us", "eu", "au", "jp", "global" + ), + ), + ), +} diff --git a/scripts/probe_reasoning.py b/scripts/probe_reasoning.py new file mode 100644 index 00000000..6588f174 --- /dev/null +++ b/scripts/probe_reasoning.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +"""Live check that per-model reasoning defaults are accepted by the providers. + +The unit and golden tests prove what CraftBot SENDS; only the provider can +say whether it ACCEPTS it. This script sends one tiny JSON request per model +through the real LLMInterface (same transports, same output cap as the app) +and reports, per model, the reasoning default that was applied and whether +the provider answered or rejected it. + +Every probe is a real, billed API call (a few hundred tokens each, more for +models that think). Nothing is sent with --dry-run. + +Usage (from the repository root): + python scripts/probe_reasoning.py # the configured LLM model + python scripts/probe_reasoning.py --model openai/gpt-5.2 --model anthropic/claude-sonnet-4-6 + python scripts/probe_reasoning.py --all-rows # every table row with credentials + python scripts/probe_reasoning.py --all-rows --dry-run + +Exit status is 1 when any probe was rejected, else 0. +""" + +from __future__ import annotations + +import argparse +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +#: Output cap of the app's LLMInterface (app/llm/interface.py), so probes +#: exercise the same reasoning output-cap logic production does. +APP_MAX_TOKENS = 8_000 + +PROBE_SYSTEM_PROMPT = ( + "You are a connectivity probe. Reply with a single JSON object and nothing else." +) +PROBE_USER_PROMPT = 'Return exactly this JSON object: {"ok": true}' + +#: Table surface -> the provider whose interface serves it. +SUBSCRIPTION_SURFACE = "openai_subscription" + + +@dataclass +class ProbeResult: + target: str + route: str + decision: str + status: str + detail: str + + +def _parse_target(value: str) -> Tuple[str, str]: + provider, sep, model = value.partition("/") + if not sep or not provider or not model: + raise argparse.ArgumentTypeError( + f"{value!r} is not / (for example openai/gpt-5.2)" + ) + return provider, model + + +def _configured_target() -> Tuple[str, str]: + from app.config import get_llm_model, get_llm_provider + + return get_llm_provider(), get_llm_model() + + +def _all_row_targets() -> List[Tuple[str, str, str]]: + """(surface, provider, model) for every row of the reasoning table.""" + from agent_core.core.models.reasoning import REASONING_RULES + + targets = [] + for surface, rows in REASONING_RULES.items(): + provider = "openai" if surface == SUBSCRIPTION_SURFACE else surface + for model in rows: + targets.append((surface, provider, model)) + return targets + + +def _has_aws_credentials() -> bool: + """Whether Bedrock would authenticate: settings keys or boto3's chain. + + boto3 builds a client without credentials and only fails on the first + call, so a missing credential must be detected up front or it would be + reported as the provider rejecting the reasoning parameters. + """ + from app.config import get_aws_credentials + + creds = get_aws_credentials() + if creds.get("access_key_id") and creds.get("secret_access_key"): + return True + import boto3 + + return boto3.Session().get_credentials() is not None + + +def _build_interface(provider: str, model: str): + """A standalone LLMInterface with the app's credentials. + + Returns ``(interface, None)``, or ``(None, reason)`` when the provider + cannot be reached with the configured credentials. + """ + from agent_core.core.impl.llm.interface import LLMInterface + from agent_core.core.models.registry import get_registry + from app.config import get_api_key, get_base_url + + profile = get_registry().get(provider) + if profile is None: + return None, f"unknown provider {provider!r}" + if profile.aws_credential_block and not _has_aws_credentials(): + return None, "no AWS credentials configured" + try: + iface = LLMInterface( + provider=provider, + model=model, + api_key=get_api_key(provider) or None, + base_url=get_base_url(provider) or None, + max_tokens=APP_MAX_TOKENS, + ) + except Exception as exc: # each provider signals missing setup its own way + return None, f"not configured: {exc}" + if not iface.is_initialized: + return None, "not configured" + return iface, None + + +def _probe( + provider: str, model: str, surface: Optional[str], dry_run: bool +) -> Optional[ProbeResult]: + target = f"{provider}/{model}" + iface, unavailable = _build_interface(provider, model) + if iface is None: + return ProbeResult(target, "-", "-", "SKIPPED", unavailable[:200]) + + from agent_core.core.models.reasoning import reasoning_surface + + route = iface._auth_mode + if surface is not None and reasoning_surface(provider, route) != surface: + # This row belongs to the other OpenAI route (API key vs ChatGPT + # subscription), which the current login cannot reach. + return None + if iface.model != model: + return ProbeResult( + target, + route, + "-", + "SKIPPED", + f"provider substitutes {iface.model!r} for this model", + ) + + decision = iface.reasoning_decision() + described = decision.describe() if decision is not None else "no rule (not sent)" + if dry_run: + return ProbeResult(target, route, described, "DRY-RUN", "nothing sent") + + started = time.perf_counter() + try: + reply = iface.generate_response( + system_prompt=PROBE_SYSTEM_PROMPT, + user_prompt=PROBE_USER_PROMPT, + log_response=False, + prompt_name="reasoning_probe", + ) + except Exception as exc: # the provider's rejection is the result + return ProbeResult(target, route, described, "REJECTED", str(exc)[:300]) + elapsed = time.perf_counter() - started + return ProbeResult( + target, route, described, "OK", f"{elapsed:.1f}s: {reply.strip()[:80]}" + ) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--model", + dest="targets", + action="append", + type=_parse_target, + default=[], + metavar="PROVIDER/MODEL", + help="probe this model (repeatable); default is the configured LLM model", + ) + parser.add_argument( + "--all-rows", + action="store_true", + help="probe every reasoning-table row whose provider has credentials", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the reasoning default per model without calling any API", + ) + args = parser.parse_args(argv) + + if args.all_rows: + plan = [(s, p, m) for s, p, m in _all_row_targets()] + elif args.targets: + plan = [(None, p, m) for p, m in args.targets] + else: + provider, model = _configured_target() + plan = [(None, provider, model)] + + results: List[ProbeResult] = [] + for surface, provider, model in plan: + result = _probe(provider, model, surface, args.dry_run) + if result is None: + continue + results.append(result) + print( + f"{result.status:9s} {result.target:55s} [{result.route}] " + f"{result.decision} :: {result.detail}", + flush=True, + ) + + rejected = [r for r in results if r.status == "REJECTED"] + ok = sum(r.status == "OK" for r in results) + skipped = sum(r.status == "SKIPPED" for r in results) + print(f"\n{ok} ok, {len(rejected)} rejected, {skipped} skipped") + return 1 if rejected else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/llm/golden/conftest.py b/tests/llm/golden/conftest.py new file mode 100644 index 00000000..90425d9a --- /dev/null +++ b/tests/llm/golden/conftest.py @@ -0,0 +1,337 @@ +# -*- coding: utf-8 -*- +"""Golden payload harness (Phase 0 of docs/PROVIDER_LAYER_CATCHUP.md). + +Freezes the EXACT request payloads LLMInterface sends per provider so the +Phase 1/2 refactors (registry consolidation, transport extraction) are +provably behavior-preserving. This is the enforcement mechanism for NFR-3 +(KV caching): cache_control placement, cachePoint position, prompt_cache_key +stability, previous_response_id chaining, and session-buffer growth are all +captured in committed JSON snapshots under tests/llm/golden/snapshots/. + +Mocking strategy: +- ``ModelFactory.create`` is patched to return a context dict with recording + fake clients, so LLMInterface's own logic (message building, cache markers, + session buffers) runs unmodified. +- BytePlus uses the REAL BytePlusCacheManager with only its network + chokepoint (``_call_responses_api``) patched, so the previous_response_id + chaining and caching-flag semantics are exercised and frozen too. +- Ollama patches the ``requests`` module reference inside interface.py. + +Snapshot policy: a missing snapshot is created and the test passes (commit +the file). An existing snapshot MUST match; drift fails the test. To +intentionally regenerate after a reviewed behavior change, run with +``GOLDEN_UPDATE=1``. +""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List + +import pytest + +SNAP_DIR = Path(__file__).resolve().parent / "snapshots" + +# ≥ 500 chars (CacheConfig.min_cache_tokens) so every caching branch fires. +GOLDEN_SYSTEM_PROMPT = ( + "You are CraftBot, a personal always-online agent. " + "You reason step by step, act through a JSON action protocol, and reply " + "with a single JSON object. " +) * 8 + +GOLDEN_TASK_ID = "task-golden" +GOLDEN_CALL_TYPE = "reasoning" +GOLDEN_SESSION_KEY = f"{GOLDEN_TASK_ID}:{GOLDEN_CALL_TYPE}" + + +class CallRecorder: + """Records every provider-bound call (client label, method, payload).""" + + def __init__(self) -> None: + self.calls: List[Dict[str, Any]] = [] + self._turn = 0 + + def next_content(self) -> str: + self._turn += 1 + return json.dumps({"turn": self._turn}) + + def record(self, client: str, method: str, payload: Dict[str, Any]) -> None: + self.calls.append( + {"client": client, "method": method, "payload": copy.deepcopy(payload)} + ) + + +# ─────────────────────────── fake clients ─────────────────────────── + + +class FakeOpenAIClient: + """Stands in for openai.OpenAI — records chat.completions.create kwargs.""" + + def __init__(self, recorder: CallRecorder) -> None: + self._recorder = recorder + self.chat = SimpleNamespace( + completions=SimpleNamespace(create=self._create) + ) + + def _create(self, **kwargs): + self._recorder.record("openai_compat", "chat.completions.create", kwargs) + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content=self._recorder.next_content()) + ) + ], + usage=SimpleNamespace( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details=SimpleNamespace(cached_tokens=0), + prompt_cache_hit_tokens=0, + ), + ) + + +class FakeAnthropicClient: + def __init__(self, recorder: CallRecorder) -> None: + self._recorder = recorder + self.messages = SimpleNamespace(create=self._create) + + def _create(self, **kwargs): + self._recorder.record("anthropic", "messages.create", kwargs) + return SimpleNamespace( + content=[SimpleNamespace(type="text", text=self._recorder.next_content())], + usage=SimpleNamespace( + input_tokens=100, + output_tokens=10, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + ) + + +class FakeBedrockClient: + def __init__(self, recorder: CallRecorder) -> None: + self._recorder = recorder + + def converse(self, **kwargs): + self._recorder.record("bedrock", "converse", kwargs) + return { + "output": { + "message": { + "content": [{"text": self._recorder.next_content()}] + } + }, + "usage": {"inputTokens": 100, "outputTokens": 10}, + } + + +def make_recording_gemini_client(recorder: CallRecorder): + """A REAL GeminiClient whose single HTTP chokepoint is a recorder. + + Recording the JSON body the client would POST (rather than the Python + kwargs the transport passes it) freezes exactly what reaches the Gemini + API, including the generationConfig the client assembles. + """ + from agent_core.core.llm.google_gemini_client import GeminiClient + + client = GeminiClient(api_key="test-gemini-key") + + def _post_json(path: str, payload: Dict[str, Any]) -> Dict[str, Any]: + recorder.record("gemini", "generateContent", {"path": path, "body": payload}) + return { + "candidates": [ + { + "content": {"parts": [{"text": recorder.next_content()}]}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "totalTokenCount": 110, + "promptTokenCount": 100, + "candidatesTokenCount": 10, + "cachedContentTokenCount": 0, + }, + } + + client._post_json = _post_json + return client + + +def make_fake_requests(recorder: CallRecorder): + """Fake for the module-level ``requests`` used by _generate_ollama.""" + + def post(url, json=None, timeout=None): # noqa: A002 - mirrors requests API + recorder.record("ollama", "requests.post", {"url": url, "json": json}) + return SimpleNamespace( + raise_for_status=lambda: None, + json=lambda: { + "response": recorder.next_content(), + "prompt_eval_count": 100, + "eval_count": 10, + }, + ) + + return SimpleNamespace(post=post) + + +def make_fake_byteplus_transport(recorder: CallRecorder): + """Replacement for BytePlusCacheManager._call_responses_api. + + The REAL manager logic (session registry, previous_response_id chaining, + caching flags) runs; only the HTTP hop is faked. Payload keys mirror the + method's signature so the chaining semantics land in the snapshot. + """ + counter = {"n": 0} + + def _call_responses_api( + self, + input_messages, + temperature, + max_tokens, + previous_response_id=None, + caching_enabled=True, + caching_prefix=False, + ): + recorder.record( + "byteplus", + "_call_responses_api", + { + "input_messages": input_messages, + "temperature": temperature, + "max_tokens": max_tokens, + "previous_response_id": previous_response_id, + "caching_enabled": caching_enabled, + "caching_prefix": caching_prefix, + }, + ) + counter["n"] += 1 + return { + "id": f"resp_{counter['n']}", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": recorder.next_content()} + ], + } + ], + "usage": { + "input_tokens": 100, + "output_tokens": 10, + "total_tokens": 110, + "input_tokens_details": {"cached_tokens": 0}, + }, + } + + return _call_responses_api + + +# ─────────────────────────── fixture ─────────────────────────── + + +def build_interface(monkeypatch, provider: str, model: str): + """Construct an LLMInterface for ``provider`` with recording fakes.""" + from agent_core.core.models.factory import ModelFactory + import agent_core.core.impl.llm.interface as interface_mod + from agent_core.core.impl.llm.cache.byteplus import BytePlusCacheManager + + recorder = CallRecorder() + + ctx: Dict[str, Any] = { + "provider": provider, + "model": model, + "client": None, + "gemini_client": None, + "anthropic_client": None, + "bedrock_client": None, + "remote_url": None, + "byteplus": None, + "initialized": True, + "auth_mode": "api_key", + } + + if provider == "anthropic": + ctx["anthropic_client"] = FakeAnthropicClient(recorder) + elif provider == "bedrock": + ctx["bedrock_client"] = FakeBedrockClient(recorder) + elif provider == "gemini": + ctx["gemini_client"] = make_recording_gemini_client(recorder) + elif provider == "byteplus": + ctx["byteplus"] = { + "api_key": "test-byteplus-key", + "base_url": "https://fake.byteplus.test/api/v3", + } + monkeypatch.setattr( + BytePlusCacheManager, + "_call_responses_api", + make_fake_byteplus_transport(recorder), + ) + elif provider == "remote": + ctx["remote_url"] = "http://localhost:11434" + # Ollama's HTTP hop lives in the chat_completions transport since + # Phase 2; patch the module reference the live code actually uses. + from agent_core.core.impl.llm.transports import chat_completions + + monkeypatch.setattr( + chat_completions, "requests", make_fake_requests(recorder) + ) + else: + # openai / deepseek / grok / openrouter / glm / fugu / minimax / moonshot + ctx["client"] = FakeOpenAIClient(recorder) + + def fake_create(**kwargs): + return dict(ctx) + + monkeypatch.setattr(ModelFactory, "create", staticmethod(fake_create)) + + iface = interface_mod.LLMInterface(provider=provider, model=model) + return iface, recorder + + +def collect_buffers(iface) -> Dict[str, Any]: + """Deep-copy the accumulated session histories (NFR-3 state). + + One interface serves one provider, so its single ``_session_histories`` + map holds whatever message shape that provider's session branch builds. + """ + return {"session_histories": copy.deepcopy(iface._session_histories)} + + +# ─────────────────────────── snapshots ─────────────────────────── + + +def assert_snapshot(name: str, data: Dict[str, Any]) -> None: + SNAP_DIR.mkdir(parents=True, exist_ok=True) + path = SNAP_DIR / f"{name}.json" + canonical = json.loads(json.dumps(data)) # normalize tuples etc. + + if os.environ.get("GOLDEN_UPDATE") == "1" or not path.exists(): + path.write_text( + json.dumps(canonical, indent=2, sort_keys=True, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + return + + expected = json.loads(path.read_text(encoding="utf-8")) + assert canonical == expected, ( + f"Golden payload drift for {name!r}.\n" + f"A request payload, cache marker, or session buffer changed. This is a " + f"behavior change by definition (see docs/PROVIDER_LAYER_CATCHUP.md " + f"section 12.3). If intentional and reviewed, regenerate with " + f"GOLDEN_UPDATE=1 and commit the diff." + ) + + +@pytest.fixture() +def golden(monkeypatch): + """Factory fixture: golden(provider, model) -> (iface, recorder).""" + + def _build(provider: str, model: str): + return build_interface(monkeypatch, provider, model) + + return _build diff --git a/tests/llm/golden/snapshots/anthropic.json b/tests/llm/golden/snapshots/anthropic.json new file mode 100644 index 00000000..e419f5a0 --- /dev/null +++ b/tests/llm/golden/snapshots/anthropic.json @@ -0,0 +1,195 @@ +{ + "calls": [ + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "max_tokens": 21000, + "messages": [ + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "claude-sonnet-4-6", + "output_config": { + "effort": "high" + }, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ], + "thinking": { + "type": "adaptive" + } + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "max_tokens": 21000, + "messages": [ + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "claude-sonnet-4-6", + "output_config": { + "effort": "high" + }, + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ], + "thinking": { + "type": "adaptive" + } + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "max_tokens": 21000, + "messages": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "{\"turn\": 2}", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "claude-sonnet-4-6", + "output_config": { + "effort": "high" + }, + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ], + "thinking": { + "type": "adaptive" + } + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "max_tokens": 21000, + "messages": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "{\"turn\": 3}", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "claude-sonnet-4-6", + "output_config": { + "effort": "high" + }, + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ], + "thinking": { + "type": "adaptive" + } + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/bedrock.json b/tests/llm/golden/snapshots/bedrock.json new file mode 100644 index 00000000..adf6686e --- /dev/null +++ b/tests/llm/golden/snapshots/bedrock.json @@ -0,0 +1,245 @@ +{ + "calls": [ + { + "client": "bedrock", + "method": "converse", + "payload": { + "additionalModelRequestFields": { + "thinking": { + "budget_tokens": 12288, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 50000 + }, + "messages": [ + { + "content": [ + { + "text": "sessionless turn" + } + ], + "role": "user" + } + ], + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "additionalModelRequestFields": { + "thinking": { + "budget_tokens": 12288, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 50000 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + } + ], + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + }, + { + "cachePoint": { + "type": "default" + } + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "additionalModelRequestFields": { + "thinking": { + "budget_tokens": 12288, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 50000 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + } + ], + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "additionalModelRequestFields": { + "thinking": { + "budget_tokens": 12288, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 50000 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 3}" + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 3" + } + ], + "role": "user" + } + ], + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 3" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 4}" + } + ], + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/byteplus.json b/tests/llm/golden/snapshots/byteplus.json new file mode 100644 index 00000000..68f6914b --- /dev/null +++ b/tests/llm/golden/snapshots/byteplus.json @@ -0,0 +1,83 @@ +{ + "calls": [ + { + "client": "byteplus", + "method": "_call_responses_api", + "payload": { + "caching_enabled": true, + "caching_prefix": false, + "input_messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "max_tokens": 50000, + "previous_response_id": null, + "temperature": 0.0 + } + }, + { + "client": "byteplus", + "method": "_call_responses_api", + "payload": { + "caching_enabled": true, + "caching_prefix": false, + "input_messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "max_tokens": 50000, + "previous_response_id": null, + "temperature": 0.0 + } + }, + { + "client": "byteplus", + "method": "_call_responses_api", + "payload": { + "caching_enabled": true, + "caching_prefix": false, + "input_messages": [ + { + "content": "session turn 2", + "role": "user" + } + ], + "max_tokens": 50000, + "previous_response_id": "resp_2", + "temperature": 0.0 + } + }, + { + "client": "byteplus", + "method": "_call_responses_api", + "payload": { + "caching_enabled": true, + "caching_prefix": false, + "input_messages": [ + { + "content": "session turn 3", + "role": "user" + } + ], + "max_tokens": 50000, + "previous_response_id": "resp_3", + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": {} + } +} diff --git a/tests/llm/golden/snapshots/deepseek.json b/tests/llm/golden/snapshots/deepseek.json new file mode 100644 index 00000000..9d481058 --- /dev/null +++ b/tests/llm/golden/snapshots/deepseek.json @@ -0,0 +1,158 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/gemini.json b/tests/llm/golden/snapshots/gemini.json new file mode 100644 index 00000000..cadf0be7 --- /dev/null +++ b/tests/llm/golden/snapshots/gemini.json @@ -0,0 +1,242 @@ +{ + "calls": [ + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "sessionless turn" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0, + "thinkingConfig": { + "thinkingBudget": 24576 + } + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.5-pro:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0, + "thinkingConfig": { + "thinkingBudget": 24576 + } + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.5-pro:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0, + "thinkingConfig": { + "thinkingBudget": 24576 + } + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.5-pro:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 3" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0, + "thinkingConfig": { + "thinkingBudget": 24576 + } + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.5-pro:generateContent" + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 3" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 4}" + } + ], + "role": "model" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/groq.json b/tests/llm/golden/snapshots/groq.json new file mode 100644 index 00000000..470d2703 --- /dev/null +++ b/tests/llm/golden/snapshots/groq.json @@ -0,0 +1,146 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "max_completion_tokens": 32768, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "llama-3.3-70b-versatile", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "max_completion_tokens": 32768, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "llama-3.3-70b-versatile", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "max_completion_tokens": 32768, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "llama-3.3-70b-versatile", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "max_completion_tokens": 32768, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "llama-3.3-70b-versatile", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/openai.json b/tests/llm/golden/snapshots/openai.json index 3a10ea09..e992d246 100644 --- a/tests/llm/golden/snapshots/openai.json +++ b/tests/llm/golden/snapshots/openai.json @@ -19,6 +19,7 @@ } ], "model": "gpt-5.2-2025-12-11", + "reasoning_effort": "high", "response_format": { "type": "json_object" } @@ -43,6 +44,7 @@ } ], "model": "gpt-5.2-2025-12-11", + "reasoning_effort": "high", "response_format": { "type": "json_object" } @@ -75,6 +77,7 @@ } ], "model": "gpt-5.2-2025-12-11", + "reasoning_effort": "high", "response_format": { "type": "json_object" } @@ -115,6 +118,7 @@ } ], "model": "gpt-5.2-2025-12-11", + "reasoning_effort": "high", "response_format": { "type": "json_object" } @@ -122,10 +126,7 @@ } ], "session_buffers": { - "anthropic": {}, - "bedrock": {}, - "gemini": {}, - "openai_compat": { + "session_histories": { "task-golden:reasoning": [ { "content": "session turn 1", @@ -152,7 +153,6 @@ "role": "assistant" } ] - }, - "openrouter_anthropic": {} + } } } diff --git a/tests/llm/golden/snapshots/openrouter_claude.json b/tests/llm/golden/snapshots/openrouter_claude.json new file mode 100644 index 00000000..e31a4903 --- /dev/null +++ b/tests/llm/golden/snapshots/openrouter_claude.json @@ -0,0 +1,181 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "type": "ephemeral" + }, + "prompt_cache_key": "907eed6c3582215b", + "reasoning": { + "max_tokens": 12288 + } + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "anthropic/claude-sonnet-4.5", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b", + "reasoning": { + "max_tokens": 12288 + } + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "anthropic/claude-sonnet-4.5", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b", + "reasoning": { + "max_tokens": 12288 + } + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "anthropic/claude-sonnet-4.5", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b", + "reasoning": { + "max_tokens": 12288 + } + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "anthropic/claude-sonnet-4.5", + "response_format": { + "type": "json_object" + } + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/openrouter_non_claude.json b/tests/llm/golden/snapshots/openrouter_non_claude.json new file mode 100644 index 00000000..0665ccc5 --- /dev/null +++ b/tests/llm/golden/snapshots/openrouter_non_claude.json @@ -0,0 +1,158 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "deepseek/deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "deepseek/deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "deepseek/deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "deepseek/deepseek-chat", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/remote.json b/tests/llm/golden/snapshots/remote.json new file mode 100644 index 00000000..f2fb10d6 --- /dev/null +++ b/tests/llm/golden/snapshots/remote.json @@ -0,0 +1,72 @@ +{ + "calls": [ + { + "client": "ollama", + "method": "requests.post", + "payload": { + "json": { + "format": "json", + "model": "llama3.2:3b", + "options": { + "temperature": 0.0 + }, + "prompt": "sessionless turn", + "stream": false, + "system": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + }, + "url": "http://localhost:11434/api/generate" + } + }, + { + "client": "ollama", + "method": "requests.post", + "payload": { + "json": { + "format": "json", + "model": "llama3.2:3b", + "options": { + "temperature": 0.0 + }, + "prompt": "session turn 1", + "stream": false + }, + "url": "http://localhost:11434/api/generate" + } + }, + { + "client": "ollama", + "method": "requests.post", + "payload": { + "json": { + "format": "json", + "model": "llama3.2:3b", + "options": { + "temperature": 0.0 + }, + "prompt": "session turn 2", + "stream": false + }, + "url": "http://localhost:11434/api/generate" + } + }, + { + "client": "ollama", + "method": "requests.post", + "payload": { + "json": { + "format": "json", + "model": "llama3.2:3b", + "options": { + "temperature": 0.0 + }, + "prompt": "session turn 3", + "stream": false + }, + "url": "http://localhost:11434/api/generate" + } + } + ], + "session_buffers": { + "session_histories": {} + } +} diff --git a/tests/llm/golden/snapshots/unruled_anthropic.json b/tests/llm/golden/snapshots/unruled_anthropic.json new file mode 100644 index 00000000..e3bdcd17 --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_anthropic.json @@ -0,0 +1,183 @@ +{ + "calls": [ + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "extra_body": { + "temperature": 0.0 + }, + "max_tokens": 16384, + "messages": [ + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "claude-3-5-haiku-20241022", + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ] + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "extra_body": { + "temperature": 0.0 + }, + "max_tokens": 16384, + "messages": [ + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "claude-3-5-haiku-20241022", + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ] + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "extra_body": { + "temperature": 0.0 + }, + "max_tokens": 16384, + "messages": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "{\"turn\": 2}", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "claude-3-5-haiku-20241022", + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ] + } + }, + { + "client": "anthropic", + "method": "messages.create", + "payload": { + "extra_body": { + "temperature": 0.0 + }, + "max_tokens": 16384, + "messages": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "{\"turn\": 3}", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "claude-3-5-haiku-20241022", + "system": [ + { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "type": "text" + } + ] + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/unruled_bedrock.json b/tests/llm/golden/snapshots/unruled_bedrock.json new file mode 100644 index 00000000..30778432 --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_bedrock.json @@ -0,0 +1,220 @@ +{ + "calls": [ + { + "client": "bedrock", + "method": "converse", + "payload": { + "inferenceConfig": { + "maxTokens": 50000, + "temperature": 0.0 + }, + "messages": [ + { + "content": [ + { + "text": "sessionless turn" + } + ], + "role": "user" + } + ], + "modelId": "meta.llama3-3-70b-instruct-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "inferenceConfig": { + "maxTokens": 50000, + "temperature": 0.0 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + } + ], + "modelId": "meta.llama3-3-70b-instruct-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "inferenceConfig": { + "maxTokens": 50000, + "temperature": 0.0 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + } + ], + "modelId": "meta.llama3-3-70b-instruct-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + { + "client": "bedrock", + "method": "converse", + "payload": { + "inferenceConfig": { + "maxTokens": 50000, + "temperature": 0.0 + }, + "messages": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 3}" + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 3" + } + ], + "role": "user" + } + ], + "modelId": "meta.llama3-3-70b-instruct-v1:0", + "system": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "session turn 3" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "{\"turn\": 4}" + } + ], + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/unruled_gemini.json b/tests/llm/golden/snapshots/unruled_gemini.json new file mode 100644 index 00000000..295ac159 --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_gemini.json @@ -0,0 +1,230 @@ +{ + "calls": [ + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "sessionless turn" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0 + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.0-flash:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0 + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.0-flash:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0 + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.0-flash:generateContent" + } + }, + { + "client": "gemini", + "method": "generateContent", + "payload": { + "body": { + "contents": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 3" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 50000, + "responseMimeType": "application/json", + "temperature": 0.0 + }, + "systemInstruction": { + "parts": [ + { + "text": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. " + } + ] + } + }, + "path": "models/gemini-2.0-flash:generateContent" + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "parts": [ + { + "text": "session turn 1" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 2}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 2" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 3}" + } + ], + "role": "model" + }, + { + "parts": [ + { + "text": "session turn 3" + } + ], + "role": "user" + }, + { + "parts": [ + { + "text": "{\"turn\": 4}" + } + ], + "role": "model" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/unruled_grok.json b/tests/llm/golden/snapshots/unruled_grok.json new file mode 100644 index 00000000..f679b356 --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_grok.json @@ -0,0 +1,158 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "grok-3", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "grok-3", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "grok-3", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "grok-3", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/unruled_openai.json b/tests/llm/golden/snapshots/unruled_openai.json new file mode 100644 index 00000000..a0ca596d --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_openai.json @@ -0,0 +1,154 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "907eed6c3582215b" + }, + "max_completion_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "gpt-4o", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_completion_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "gpt-4o", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_completion_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "gpt-4o", + "response_format": { + "type": "json_object" + } + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_completion_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "gpt-4o", + "response_format": { + "type": "json_object" + } + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/snapshots/unruled_openrouter_claude.json b/tests/llm/golden/snapshots/unruled_openrouter_claude.json new file mode 100644 index 00000000..4214b7f7 --- /dev/null +++ b/tests/llm/golden/snapshots/unruled_openrouter_claude.json @@ -0,0 +1,173 @@ +{ + "calls": [ + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "type": "ephemeral" + }, + "prompt_cache_key": "907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "sessionless turn", + "role": "user" + } + ], + "model": "anthropic/claude-3.5-sonnet", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + } + ], + "model": "anthropic/claude-3.5-sonnet", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + } + ], + "model": "anthropic/claude-3.5-sonnet", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + }, + { + "client": "openai_compat", + "method": "chat.completions.create", + "payload": { + "extra_body": { + "cache_control": { + "ttl": "1h", + "type": "ephemeral" + }, + "prompt_cache_key": "reasoning_907eed6c3582215b" + }, + "max_tokens": 50000, + "messages": [ + { + "content": "You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. You are CraftBot, a personal always-online agent. You reason step by step, act through a JSON action protocol, and reply with a single JSON object. ", + "role": "system" + }, + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + } + ], + "model": "anthropic/claude-3.5-sonnet", + "response_format": { + "type": "json_object" + }, + "temperature": 0.0 + } + } + ], + "session_buffers": { + "session_histories": { + "task-golden:reasoning": [ + { + "content": "session turn 1", + "role": "user" + }, + { + "content": "{\"turn\": 2}", + "role": "assistant" + }, + { + "content": "session turn 2", + "role": "user" + }, + { + "content": "{\"turn\": 3}", + "role": "assistant" + }, + { + "content": "session turn 3", + "role": "user" + }, + { + "content": "{\"turn\": 4}", + "role": "assistant" + } + ] + } + } +} diff --git a/tests/llm/golden/test_golden_payloads.py b/tests/llm/golden/test_golden_payloads.py index f15b8846..0225e515 100644 --- a/tests/llm/golden/test_golden_payloads.py +++ b/tests/llm/golden/test_golden_payloads.py @@ -12,8 +12,11 @@ from __future__ import annotations +import pytest + from .conftest import ( GOLDEN_CALL_TYPE, + GOLDEN_SESSION_KEY, GOLDEN_SYSTEM_PROMPT, GOLDEN_TASK_ID, assert_snapshot, @@ -94,6 +97,16 @@ def test_anthropic(golden): assert marked[0][0] == 3 # index of the last assistant message assert marked[0][1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + # Reasoning default (Claude 4.6 row): adaptive thinking at effort high, + # temperature dropped (incompatible with thinking), max_tokens raised + # for thinking but kept under the SDK's non-streaming ceiling. + for call in creates: + payload = call["payload"] + assert payload["thinking"] == {"type": "adaptive"} + assert payload["output_config"] == {"effort": "high"} + assert "extra_body" not in payload + assert payload["max_tokens"] == 21000 + snapshot_scenario("anthropic", iface, rec) @@ -137,6 +150,16 @@ def test_bedrock(golden): assert i == 3 # last assistant in [u1, a1, u2, a2, new_user] assert j == len(turn3["messages"][i]["content"]) - 1 + # Reasoning default (Claude 4.5 budget row via Converse): budget thinking + # at the "high" rung, temperature dropped, maxTokens never lowered. + for call in converses: + payload = call["payload"] + assert payload["additionalModelRequestFields"] == { + "thinking": {"type": "enabled", "budget_tokens": 12288} + } + assert "temperature" not in payload["inferenceConfig"] + assert payload["inferenceConfig"]["maxTokens"] == 50000 + snapshot_scenario("bedrock", iface, rec) @@ -175,6 +198,11 @@ def test_openai(golden): assert msgs[0]["role"] == "system" assert msgs[-1]["role"] == "user" + # Reasoning default (gpt-5.2 row): one level below xhigh. The output cap + # is never lowered (this fixture's max_tokens already exceeds the row's). + assert all(c["payload"]["reasoning_effort"] == "high" for c in creates) + assert all(c["payload"]["max_completion_tokens"] == 50000 for c in creates) + snapshot_scenario("openai", iface, rec) @@ -330,11 +358,18 @@ def test_openrouter_claude(golden): assert extra["cache_control"] == {"type": "ephemeral", "ttl": "1h"} assert extra["prompt_cache_key"].startswith(f"{GOLDEN_CALL_TYPE}_") - # Accumulation must use the OpenRouter-Anthropic buffer, not openai_compat. - buffers = collect_buffers(iface) - key = f"{GOLDEN_TASK_ID}:{GOLDEN_CALL_TYPE}" - assert len(buffers["openrouter_anthropic"][key]) == 6 # 3 turns x (u, a) - assert buffers["openai_compat"] == {} + # Accumulation: 3 turns x (user, assistant) in OpenAI message shape. + history = collect_buffers(iface)["session_histories"][GOLDEN_SESSION_KEY] + assert [m["role"] for m in history] == ["user", "assistant"] * 3 + + # Reasoning default (Claude 4.5 budget row via OpenRouter): a fixed + # thinking budget beside cache_control, temperature dropped, and the + # request max_tokens strictly above the budget. + for call in creates: + payload = call["payload"] + assert payload["extra_body"]["reasoning"] == {"max_tokens": 12288} + assert "temperature" not in payload + assert payload["max_tokens"] > 12288 snapshot_scenario("openrouter_claude", iface, rec) @@ -351,10 +386,8 @@ def test_openrouter_non_claude(golden): extra = call["payload"].get("extra_body", {}) assert "cache_control" not in extra - buffers = collect_buffers(iface) - key = f"{GOLDEN_TASK_ID}:{GOLDEN_CALL_TYPE}" - assert len(buffers["openai_compat"][key]) == 6 - assert buffers["openrouter_anthropic"] == {} + history = collect_buffers(iface)["session_histories"][GOLDEN_SESSION_KEY] + assert [m["role"] for m in history] == ["user", "assistant"] * 3 snapshot_scenario("openrouter_non_claude", iface, rec) @@ -373,9 +406,8 @@ def test_groq_new_provider(golden): assert "prompt_cache_key" not in extra assert "cache_control" not in extra - buffers = collect_buffers(iface) - key = f"{GOLDEN_TASK_ID}:{GOLDEN_CALL_TYPE}" - assert len(buffers["openai_compat"][key]) == 6 # 3 turns x (u, a) + history = collect_buffers(iface)["session_histories"][GOLDEN_SESSION_KEY] + assert len(history) == 6 # 3 turns x (u, a) snapshot_scenario("groq", iface, rec) @@ -387,17 +419,28 @@ def test_gemini(golden): iface, rec = golden("gemini", "gemini-2.5-pro") run_scenario(iface) - # Sessionless (no call_type) -> single-turn generate_text. - assert rec.calls[0]["method"] == "generate_text" + posts = [c for c in rec.calls if c["method"] == "generateContent"] + assert len(posts) == 4 + assert all( + p["payload"]["path"] == "models/gemini-2.5-pro:generateContent" for p in posts + ) + + # Sessionless (no call_type) -> single-turn request. + assert len(posts[0]["payload"]["body"]["contents"]) == 1 # Session turns -> multiturn contents array growing by 2 per turn. - multiturns = [c for c in rec.calls if c["method"] == "generate_text_multiturn"] - assert len(multiturns) == 3 - for n, call in enumerate(multiturns, start=1): - contents = call["payload"]["contents"] + for n, call in enumerate(posts[1:], start=1): + contents = call["payload"]["body"]["contents"] assert len(contents) == 2 * (n - 1) + 1 assert contents[-1]["role"] == "user" + # Reasoning default (gemini-2.5-pro budget row) on both the single-turn + # and the multi-turn request. + for post in posts: + config = post["payload"]["body"]["generationConfig"] + assert config["thinkingConfig"] == {"thinkingBudget": 24576} + assert config["maxOutputTokens"] >= 24576 + 8192 + snapshot_scenario("gemini", iface, rec) @@ -447,3 +490,27 @@ def test_remote_ollama(golden): assert "system" not in posts[3]["payload"]["json"] snapshot_scenario("remote", iface, rec) + + +# ───────────────────── Models without a reasoning rule ───────────────────── +# +# A model the reasoning table (agent_core/core/models/reasoning.py) does not +# list must keep receiving exactly the payload it received before reasoning +# defaults existed. These snapshots were recorded before that feature landed +# and must never drift: one scenario per wire that the feature touches. + +UNRULED_MODELS = [ + ("openai", "gpt-4o", "unruled_openai"), + ("anthropic", "claude-3-5-haiku-20241022", "unruled_anthropic"), + ("bedrock", "meta.llama3-3-70b-instruct-v1:0", "unruled_bedrock"), + ("gemini", "gemini-2.0-flash", "unruled_gemini"), + ("openrouter", "anthropic/claude-3.5-sonnet", "unruled_openrouter_claude"), + ("grok", "grok-3", "unruled_grok"), +] + + +@pytest.mark.parametrize("provider, model, snapshot", UNRULED_MODELS) +def test_unruled_model_payload_is_unchanged(golden, provider, model, snapshot): + iface, rec = golden(provider, model) + run_scenario(iface) + snapshot_scenario(snapshot, iface, rec) diff --git a/tests/llm/test_reasoning_rules.py b/tests/llm/test_reasoning_rules.py new file mode 100644 index 00000000..8a45f976 --- /dev/null +++ b/tests/llm/test_reasoning_rules.py @@ -0,0 +1,449 @@ +# -*- coding: utf-8 -*- +"""Per-model reasoning defaults (agent_core/core/models/reasoning.py). + +Four layers: + +1. The table: every row is well formed, uses only the levels its wire can + carry, and is filed under a provider whose transport can send it. +2. The level policy: one level below the strongest, never above "high", + and never below the provider's own default (then the strongest level). +3. Wire rendering: each decision becomes exactly its provider's fields. +4. The transports at CraftBot's real settings (max_tokens 8000): a model with + a rule gets its fields, output cap, and temperature handling; a model + without one is untouched (the golden unruled_* snapshots additionally pin + those payloads byte for byte). +""" + +from __future__ import annotations + +import pytest + +from agent_core.core.impl.llm import reasoning_wire +from agent_core.core.impl.llm.interface import LLMContextOverflowError +from agent_core.core.impl.llm.reasoning_wire import TRANSPORT_WIRES +from agent_core.core.llm.google_gemini_client import _thinking_config +from agent_core.core.models.chatgpt_subscription_client import _translate_request +from agent_core.core.models.reasoning import ( + BUDGET_RUNGS, + BUDGET_WIRES, + KNOWN_LEVELS, + LEVEL_CEILING, + REASONING_RULES, + ReasoningDecision, + ReasoningRule, + ReasoningWire, + resolve_reasoning, + target_level, +) +from agent_core.core.models.registry import get_registry + +from .golden.conftest import GOLDEN_SYSTEM_PROMPT, build_interface + +#: The Anthropic SDK refuses non-streaming requests above this max_tokens. +ANTHROPIC_NON_STREAMING_CEILING = 21_333 +#: Anthropic and OpenRouter reject thinking budgets below this. +MIN_THINKING_BUDGET = 1_024 +#: CraftBot's app-level LLMInterface output cap (app/llm/interface.py). +APP_MAX_TOKENS = 8_000 + + +def _provider(surface: str) -> str: + return "openai" if surface == "openai_subscription" else surface + + +def _auth_mode(surface: str) -> str: + return "subscription" if surface == "openai_subscription" else "api_key" + + +ROWS = [ + (surface, model, rule) + for surface, rows in REASONING_RULES.items() + for model, rule in rows.items() +] +ROW_IDS = [f"{surface}/{model}" for surface, model, _ in ROWS] + + +# ─────────────────────────────── 1. the table ─────────────────────────────── + + +@pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) +def test_row_is_well_formed(surface, model, rule): + assert model == model.strip() and model + assert rule.levels, "a rule needs at least one level" + assert len(set(rule.levels)) == len(rule.levels) + assert all(level in KNOWN_LEVELS for level in rule.levels) + assert list(rule.levels) == sorted(rule.levels, key=KNOWN_LEVELS.index) + assert rule.provider_default is None or rule.provider_default in rule.levels + assert rule.source.startswith("https://") + assert rule.output_tokens >= 0 + if rule.wire in BUDGET_WIRES: + assert rule.levels == BUDGET_RUNGS + assert len(rule.budgets) == len(rule.levels) + assert list(rule.budgets) == sorted(set(rule.budgets)) + assert rule.budgets[0] >= MIN_THINKING_BUDGET + else: + assert rule.budgets == () + + +@pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) +def test_row_is_sendable_by_its_providers_transport(surface, model, rule): + profile = get_registry().get(_provider(surface)) + assert profile is not None, f"{surface!r} is not a registered provider" + assert rule.wire in TRANSPORT_WIRES[profile.wire] + + +@pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) +def test_anthropic_caps_stay_below_the_sdk_non_streaming_ceiling(surface, model, rule): + if rule.wire in (ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET): + assert 0 < rule.output_tokens < ANTHROPIC_NON_STREAMING_CEILING + + +def test_duplicate_model_ids_are_refused(): + from agent_core.core.models import reasoning + + rule = REASONING_RULES["openai"]["gpt-5.2"] + with pytest.raises(ValueError, match="listed twice"): + reasoning._surface({"m": rule}, {"m": rule}) + + +# ─────────────────────────────── 2. the policy ────────────────────────────── + + +@pytest.mark.parametrize( + "levels, provider_default, expected", + [ + (("low", "medium", "high"), None, "medium"), + (("low", "medium", "high", "xhigh"), None, "high"), + # xhigh would be one below max; the ceiling keeps it at high. + (("low", "medium", "high", "xhigh", "max"), "medium", "high"), + (("low", "medium", "high", "max"), "high", "high"), + # medium would be below the provider's own high: use the strongest. + (("low", "medium", "high"), "high", "high"), + (("low", "high", "max"), "max", "max"), + (("high", "max"), "max", "max"), + (("low", "high", "max"), "high", "high"), + (("high",), None, "high"), + (BUDGET_RUNGS, None, "high"), + ], +) +def test_target_level_policy(levels, provider_default, expected): + rule = ReasoningRule( + wire=ReasoningWire.EFFORT, + levels=levels, + provider_default=provider_default, + source="https://example.test", + ) + assert target_level(rule) == expected + + +@pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) +def test_every_row_resolves_within_the_policy(surface, model, rule): + decision = resolve_reasoning(_provider(surface), model, _auth_mode(surface)) + assert decision is not None + assert decision.key == f"{surface}/{model}" + chosen = rule.levels.index(decision.level) + if rule.provider_default is not None: + # Never weaker than what the provider applies when it is omitted. + assert chosen >= rule.levels.index(rule.provider_default) + if LEVEL_CEILING in rule.levels and chosen > rule.levels.index(LEVEL_CEILING): + # Above the ceiling only when the provider's own default already is. + assert rule.provider_default is not None + assert rule.levels.index(rule.provider_default) > rule.levels.index( + LEVEL_CEILING + ) + assert chosen == len(rule.levels) - 1 + if decision.budget_tokens is not None: + assert decision.budget_tokens < decision.output_tokens + + +@pytest.mark.parametrize( + "provider, model, auth_mode, expected", + [ + ("openai", "gpt-5.2-2025-12-11", "api_key", "high"), # CraftBot's default + ("openai", "gpt-5.1", "api_key", "medium"), + ("openai", "gpt-5", "api_key", "medium"), + ("openai", "gpt-6-sol", "api_key", "high"), # never xhigh/max + ("openai", "gpt-5.5", "subscription", "high"), + ("openai", "gpt-6.1-sol", "subscription", "high"), + ("anthropic", "claude-sonnet-4-6", "api_key", "high"), + ("anthropic", "claude-opus-5-5", "api_key", "high"), + ("anthropic", "claude-fable-5-1", "api_key", "high"), + ("openrouter", "openai/gpt-5.2", "api_key", "high"), + ("gemini", "gemini-3.5-flash", "api_key", "medium"), + ("gemini", "gemini-3.1-pro-preview", "api_key", "high"), + ("gemini", "gemini-3-flash-preview", "api_key", "high"), + ("grok", "grok-4.3", "api_key", "high"), + ("grok", "grok-4.5", "api_key", "high"), + ("grok", "grok-4.7", "api_key", "high"), + ("deepseek", "deepseek-flash", "api_key", "high"), + ("glm", "glm-5.3", "api_key", "max"), + ("glm", "glm-5.2", "api_key", "max"), + ("moonshot", "kimi-k3", "api_key", "max"), + ("groq", "openai/gpt-oss-120b", "api_key", "medium"), + ("cerebras", "gpt-oss-120b", "api_key", "medium"), + ], +) +def test_resolved_levels(provider, model, auth_mode, expected): + assert resolve_reasoning(provider, model, auth_mode).level == expected + + +@pytest.mark.parametrize( + "provider, model, budget", + [ + ("anthropic", "claude-haiku-4-5-20251001", 12_288), + ("bedrock", "us.anthropic.claude-haiku-4-5-20251001-v1:0", 12_288), + ("openrouter", "anthropic/claude-sonnet-4.5", 12_288), + ("gemini", "gemini-2.5-pro", 24_576), + ("gemini", "gemini-2.5-flash", 18_432), + ], +) +def test_budget_models_resolve_to_the_high_rung(provider, model, budget): + decision = resolve_reasoning(provider, model) + assert decision.level == "high" + assert decision.budget_tokens == budget + + +@pytest.mark.parametrize( + "provider, model, auth_mode", + [ + ("openai", "gpt-4o", "api_key"), + ("openai", "gpt-5.2", "subscription"), # not in the Codex catalogue + ("openai", "gpt-5.5", "api_key_typo"), # unknown auth mode: api surface + ("openai", "GPT-5.2", "api_key"), # ids are exact + ("openai", " gpt-5.2", "api_key"), + ("openai", "gpt-5.2-pro", "api_key"), + ("grok", "grok-3", "api_key"), + ("mistral", "mistral-large-latest", "api_key"), + ("qwen", "qwen-max", "api_key"), + ("remote", "llama3.2:3b", "api_key"), + ("", "gpt-5.2", "api_key"), + ("openai", None, "api_key"), + (None, "gpt-5.2", "api_key"), + ], +) +def test_models_without_a_rule_resolve_to_none(provider, model, auth_mode): + if auth_mode == "api_key_typo": + # A non-subscription auth mode reads the public-API surface. + assert resolve_reasoning(provider, model, auth_mode).key == "openai/gpt-5.5" + return + assert resolve_reasoning(provider, model, auth_mode) is None + + +def test_output_cap_never_lowers_the_callers_cap(): + decision = resolve_reasoning("openai", "gpt-5.2") + assert decision.output_cap(APP_MAX_TOKENS) == 32_000 + assert decision.output_cap(50_000) == 50_000 + + +# ─────────────────────────────── 3. rendering ─────────────────────────────── + + +def _decision(wire, level="high", budget=None): + return ReasoningDecision( + key="test/model", + wire=wire, + level=level, + budget_tokens=budget, + output_tokens=0, + omit_temperature=False, + ) + + +def test_chat_completions_fields(): + assert reasoning_wire.chat_completions_fields(_decision(ReasoningWire.EFFORT)) == ( + {"reasoning_effort": "high"}, + {}, + ) + assert reasoning_wire.chat_completions_fields( + _decision(ReasoningWire.OPENROUTER_EFFORT) + ) == ({}, {"reasoning": {"effort": "high"}}) + assert reasoning_wire.chat_completions_fields( + _decision(ReasoningWire.OPENROUTER_BUDGET, budget=12_288) + ) == ({}, {"reasoning": {"max_tokens": 12_288}}) + + +def test_anthropic_fields(): + assert reasoning_wire.anthropic_fields( + _decision(ReasoningWire.ANTHROPIC_ADAPTIVE) + ) == {"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}} + assert reasoning_wire.anthropic_fields( + _decision(ReasoningWire.ANTHROPIC_BUDGET, budget=12_288) + ) == {"thinking": {"type": "enabled", "budget_tokens": 12_288}} + + +def test_gemini_thinking_kwargs(): + assert reasoning_wire.gemini_thinking_kwargs( + _decision(ReasoningWire.GEMINI_LEVEL, level="medium") + ) == {"thinking_level": "medium"} + assert reasoning_wire.gemini_thinking_kwargs( + _decision(ReasoningWire.GEMINI_BUDGET, budget=24_576) + ) == {"thinking_budget": 24_576} + + +@pytest.mark.parametrize( + "render, wire", + [ + (reasoning_wire.chat_completions_fields, ReasoningWire.ANTHROPIC_ADAPTIVE), + (reasoning_wire.anthropic_fields, ReasoningWire.EFFORT), + (reasoning_wire.gemini_thinking_kwargs, ReasoningWire.OPENROUTER_EFFORT), + ], +) +def test_a_wire_the_transport_cannot_send_raises(render, wire): + with pytest.raises(ValueError, match="wrong provider"): + render(_decision(wire)) + + +def test_gemini_thinking_config_builder(): + assert _thinking_config(None, None) is None + assert _thinking_config(512, None) == {"thinkingBudget": 512} + assert _thinking_config(None, "medium") == {"thinkingLevel": "medium"} + with pytest.raises(ValueError, match="not both"): + _thinking_config(512, "medium") + + +def test_codex_translator_uses_the_resolved_effort(): + messages = [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}] + ruled = _translate_request( + {"model": "gpt-5.5", "messages": messages, "reasoning_effort": "high"}, "k" + ) + assert ruled["reasoning"] == {"effort": "high", "summary": "auto"} + # No rule: the backend still requires the block, so today's medium stays. + unruled = _translate_request({"model": "gpt-5.4", "messages": messages}, "k") + assert unruled["reasoning"] == {"effort": "medium", "summary": "auto"} + + +# ────────────────────── 4. transports at CraftBot's settings ───────────────── + + +def _one_call(monkeypatch, provider, model, auth_mode="api_key"): + iface, rec = build_interface(monkeypatch, provider, model) + iface.max_tokens = APP_MAX_TOKENS + iface._auth_mode = auth_mode + iface.generate_response(system_prompt=GOLDEN_SYSTEM_PROMPT, user_prompt="hi") + assert len(rec.calls) == 1 + return iface, rec.calls[0]["payload"] + + +def test_openai_ruled_model(monkeypatch): + _, payload = _one_call(monkeypatch, "openai", "gpt-5.2-2025-12-11") + assert payload["reasoning_effort"] == "high" + assert payload["max_completion_tokens"] == 32_000 + assert "temperature" not in payload + assert payload["response_format"] == {"type": "json_object"} + + +def test_openai_unruled_model(monkeypatch): + _, payload = _one_call(monkeypatch, "openai", "gpt-4o") + assert "reasoning_effort" not in payload + assert payload["max_completion_tokens"] == APP_MAX_TOKENS + + +def test_subscription_request_carries_the_codex_row(monkeypatch): + _, payload = _one_call(monkeypatch, "openai", "gpt-5.5", auth_mode="subscription") + assert payload["reasoning_effort"] == "high" + # Codex rows leave the (dropped) output cap alone. + assert payload["max_completion_tokens"] == APP_MAX_TOKENS + assert _translate_request(payload, "k")["reasoning"]["effort"] == "high" + + +def test_groq_cap_is_clamped_by_the_profile_and_temperature_kept(monkeypatch): + _, payload = _one_call(monkeypatch, "groq", "openai/gpt-oss-120b") + assert payload["reasoning_effort"] == "medium" + assert payload["max_completion_tokens"] == 32_000 # <= Groq's 32,768 clamp + assert payload["temperature"] == 0.0 + assert payload["response_format"] == {"type": "json_object"} + + +def test_cerebras_cap_is_clamped_by_the_profile(monkeypatch): + _, payload = _one_call(monkeypatch, "cerebras", "gpt-oss-120b") + assert payload["reasoning_effort"] == "medium" + assert payload["max_completion_tokens"] == 32_000 + + +def test_glm_bad_model_is_pinned_at_its_max(monkeypatch): + _, payload = _one_call(monkeypatch, "glm", "glm-5.3") + assert payload["reasoning_effort"] == "max" + assert payload["max_tokens"] == 32_000 + + +def test_openrouter_openai_row(monkeypatch): + _, payload = _one_call(monkeypatch, "openrouter", "openai/gpt-5.2") + assert payload["extra_body"]["reasoning"] == {"effort": "high"} + assert "reasoning_effort" not in payload + assert "temperature" not in payload + assert payload["max_tokens"] == 32_000 + + +def test_anthropic_ruled_model(monkeypatch): + _, payload = _one_call(monkeypatch, "anthropic", "claude-opus-5-5") + assert payload["thinking"] == {"type": "adaptive"} + assert payload["output_config"] == {"effort": "high"} + assert payload["max_tokens"] == 21_000 + assert "extra_body" not in payload # no temperature + + +def test_anthropic_unruled_model(monkeypatch): + _, payload = _one_call(monkeypatch, "anthropic", "claude-3-5-haiku-20241022") + assert "thinking" not in payload and "output_config" not in payload + assert payload["max_tokens"] == 16_384 + assert payload["extra_body"] == {"temperature": 0.0} + + +def test_bedrock_adaptive_row(monkeypatch): + _, payload = _one_call(monkeypatch, "bedrock", "us.anthropic.claude-opus-4-8") + assert payload["additionalModelRequestFields"] == { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "high"}, + } + assert payload["inferenceConfig"] == {"maxTokens": 21_000} + + +def test_bedrock_unruled_model(monkeypatch): + _, payload = _one_call(monkeypatch, "bedrock", "meta.llama3-3-70b-instruct-v1:0") + assert "additionalModelRequestFields" not in payload + assert payload["inferenceConfig"] == { + "temperature": 0.0, + "maxTokens": APP_MAX_TOKENS, + } + + +def test_gemini_level_row(monkeypatch): + _, payload = _one_call(monkeypatch, "gemini", "gemini-3.5-flash") + config = payload["body"]["generationConfig"] + assert config["thinkingConfig"] == {"thinkingLevel": "medium"} + assert config["maxOutputTokens"] == 32_768 + + +def test_gemini_caller_budget_wins_over_the_default(monkeypatch): + iface, rec = build_interface(monkeypatch, "gemini", "gemini-3.5-flash") + iface.max_tokens = APP_MAX_TOKENS + iface._begin_call(thinking_budget=512) + iface._generate_response_sync(GOLDEN_SYSTEM_PROMPT, "hi") + config = rec.calls[0]["payload"]["body"]["generationConfig"] + assert config["thinkingConfig"] == {"thinkingBudget": 512} + assert config["maxOutputTokens"] == APP_MAX_TOKENS + + +def test_context_check_reserves_the_reasoning_cap(monkeypatch): + import app.config as app_config + + monkeypatch.setattr(app_config, "get_context_window", lambda: 40_000) + prompt = "word " * 12_000 # ~12k tokens: fits 40k - 8k, not 40k - 32k + + unruled, _ = build_interface(monkeypatch, "openai", "gpt-4o") + unruled.max_tokens = APP_MAX_TOKENS + assert unruled._output_reservation() == APP_MAX_TOKENS + unruled._check_context_fits(None, prompt) + + ruled, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + ruled.max_tokens = APP_MAX_TOKENS + assert ruled._output_reservation() == 32_000 + with pytest.raises(LLMContextOverflowError, match="32000 reserved for output"): + ruled._check_context_fits(None, prompt) + + +def test_decision_follows_a_model_switch(monkeypatch): + iface, _ = build_interface(monkeypatch, "openai", "gpt-4o") + assert iface.reasoning_decision() is None + iface.model = "gpt-5.2" + assert iface.reasoning_decision().level == "high" diff --git a/tests/llm/test_reliability.py b/tests/llm/test_reliability.py new file mode 100644 index 00000000..36006e56 --- /dev/null +++ b/tests/llm/test_reliability.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +"""Phase 5 reliability tests: credential pools (FR-7) and cross-provider +fallback (FR-9). + +Fallback assertions include the NFR-3 property from the spec's Phase 5 +acceptance: after a fallback turn, the PRIMARY interface's session buffers +are untouched and the fallback interface accumulates its own, so returning +to the primary next turn stays cache-warm. +""" + +from __future__ import annotations + +import pytest + +from agent_core.core.impl.llm.errors import LLMConsecutiveFailureError + +from .golden.conftest import ( + GOLDEN_SYSTEM_PROMPT, + build_interface, +) + + +# ─────────────────────── credential pool state machine ─────────────────────── + + +@pytest.fixture() +def pool(monkeypatch, tmp_path): + from agent_core.core.models import credentials + + monkeypatch.setattr( + credentials, "_state_path", lambda: tmp_path / "pool_state.json" + ) + credentials.reset_state_for_tests() + + keys = {"primary": "key-A", "extras": ["key-B", "key-C"]} + import app.config as app_config + + monkeypatch.setattr(app_config, "get_api_key", lambda p: keys["primary"]) + monkeypatch.setattr( + app_config, "get_extra_api_keys", lambda p: list(keys["extras"]) + ) + return credentials + + +def test_fill_first_serves_primary(pool): + assert pool.resolve("anthropic") == "key-A" + assert pool.has_pool("anthropic") is True + + +def test_rate_limit_cools_after_second_consecutive(pool): + pool.resolve("anthropic") + pool.note_failure("anthropic", "rate_limit") # 1st: no cooldown yet + assert pool.resolve("anthropic") == "key-A" + pool.note_failure("anthropic", "rate_limit") # 2nd: cool 60s + assert pool.resolve("anthropic") == "key-B" + + +def test_billing_rotates_immediately(pool): + pool.resolve("anthropic") + pool.note_failure("anthropic", "credit") + assert pool.resolve("anthropic") == "key-B" + pool.note_failure("anthropic", "credit") # B billing-cooled too + assert pool.resolve("anthropic") == "key-C" + + +def test_auth_rotates_and_success_clears(pool): + pool.resolve("anthropic") + pool.note_failure("anthropic", "auth") + assert pool.resolve("anthropic") == "key-B" + pool.note_success("anthropic") # clears B's (empty) record, keeps A cooling + assert pool.resolve("anthropic") == "key-B" + + +def test_all_cooling_fails_open_to_primary(pool): + for _ in range(3): + pool.resolve("anthropic") + pool.note_failure("anthropic", "credit") + assert pool.resolve("anthropic") == "key-A" + + +def test_transient_categories_do_not_cool(pool): + pool.resolve("anthropic") + pool.note_failure("anthropic", "server") + pool.note_failure("anthropic", "connection") + pool.note_failure("anthropic", "unknown") + assert pool.resolve("anthropic") == "key-A" + + +def test_single_key_provider_is_untouched(pool, monkeypatch): + import app.config as app_config + + monkeypatch.setattr(app_config, "get_extra_api_keys", lambda p: []) + assert pool.has_pool("anthropic") is False + pool.resolve("anthropic") + pool.note_failure("anthropic", "credit") + assert pool.resolve("anthropic") == "key-A" + + +# ─────────────────────── cross-provider fallback ─────────────────────── + + +def _break_client(iface): + """Make the fake OpenAI-compat client raise on every call.""" + + def failing_create(**kwargs): + raise RuntimeError("primary provider down") + + iface.client.chat = type(iface.client.chat)( + completions=type(iface.client.chat.completions)(create=failing_create) + ) + + +@pytest.fixture() +def fallback_pair(monkeypatch): + """Primary openai interface (broken) + anthropic fallback (healthy), + with the chain configured and the fallback interface injected.""" + primary, primary_rec = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + fallback, fallback_rec = build_interface( + monkeypatch, "anthropic", "claude-sonnet-4-6" + ) + _break_client(primary) + + import app.config as app_config + + monkeypatch.setattr( + app_config, "get_fallback_providers", lambda: ["anthropic"] + ) + monkeypatch.setattr(app_config, "get_api_key", lambda p: "k") + # Inject the pre-built fallback interface (its factory patch would + # otherwise have been torn down by build_interface's second call). + primary._fallback_interfaces["anthropic"] = fallback + return primary, primary_rec, fallback, fallback_rec + + +def test_fallback_serves_sessionless_turn(fallback_pair): + primary, _prec, fallback, frec = fallback_pair + + out = primary.generate_response( + system_prompt=GOLDEN_SYSTEM_PROMPT, user_prompt="hello" + ) + assert out # turn served by the fallback + assert any(c["method"] == "messages.create" for c in frec.calls) + # Served turn == success: the primary's counter must not tick. + assert primary.consecutive_failures == 0 + + +def test_fallback_session_turn_preserves_primary_buffers(fallback_pair): + primary, _prec, fallback, frec = fallback_pair + + # Prime one successful-looking primary session state by hand: the + # buffers below must survive the fallback turn untouched. + primary.create_session_cache("t1", "reasoning", GOLDEN_SYSTEM_PROMPT) + primary._session_histories["t1:reasoning"] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + before = [dict(m) for m in primary._session_histories["t1:reasoning"]] + + out = primary.generate_response_with_session("t1", "reasoning", "turn 2") + assert out # served by anthropic fallback + + # NFR-3: primary buffers untouched; fallback accumulated its own. + assert primary._session_histories["t1:reasoning"] == before + assert fallback._session_histories["t1:reasoning"] + assert primary.consecutive_failures == 0 + + +def test_fallback_off_preserves_error_contract(monkeypatch): + """No chain configured -> exact historical failure behavior (NFR-1).""" + primary, rec = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + _break_client(primary) + import app.config as app_config + + monkeypatch.setattr(app_config, "get_fallback_providers", lambda: []) + + for _ in range(4): + with pytest.raises(Exception): + primary.generate_response(system_prompt="s", user_prompt="u") + assert primary.consecutive_failures == 4 + with pytest.raises(LLMConsecutiveFailureError): + primary.generate_response(system_prompt="s", user_prompt="u") + + +def test_strict_turn_after_reinitialize_skips_fallback(fallback_pair): + primary, _prec, fallback, frec = fallback_pair + + # Simulate the flag reinitialize() sets on explicit selection. + primary._suppress_fallback_once = True + with pytest.raises(Exception): + primary.generate_response(system_prompt="s", user_prompt="u") + assert not any(c["method"] == "messages.create" for c in frec.calls) + + # Next turn: fallback active again. + out = primary.generate_response(system_prompt="s", user_prompt="u") + assert out + assert any(c["method"] == "messages.create" for c in frec.calls) + + +def test_total_outage_terminates_without_recursion(monkeypatch): + """Multi-provider chain with EVERYTHING down must terminate in one chain + walk (audit finding 2026-08-17: fallback instances used to walk their + own chains, nesting primary -> fb -> fb-of-fb without bound).""" + primary, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + fb1, _ = build_interface(monkeypatch, "anthropic", "claude-sonnet-4-6") + fb2, _ = build_interface(monkeypatch, "deepseek", "deepseek-chat") + _break_client(primary) + fb1.messages_broken = True + + def failing_messages(**kwargs): + raise RuntimeError("anthropic down") + + fb1._anthropic_client.messages = type(fb1._anthropic_client.messages)( + create=failing_messages + ) + _break_client(fb2) + + import app.config as app_config + + monkeypatch.setattr( + app_config, "get_fallback_providers", lambda: ["anthropic", "deepseek"] + ) + monkeypatch.setattr(app_config, "get_api_key", lambda p: "k") + primary._fallback_interfaces = {"anthropic": fb1, "deepseek": fb2} + fb1._is_fallback_instance = True + fb2._is_fallback_instance = True + + with pytest.raises(Exception) as exc_info: + primary.generate_response(system_prompt="s", user_prompt="u") + # Terminated with a plain failure, not a RecursionError. + assert not isinstance(exc_info.value, RecursionError) + assert primary.consecutive_failures == 1 + + # Structural guarantee: fallback instances never expose a chain. + assert fb1._fallback_chain() == [] + assert fb2._fallback_chain() == [] + + +def test_fallback_notice_hook_fires(fallback_pair): + primary, _prec, fallback, _frec = fallback_pair + events = [] + primary._on_fallback = lambda frm, to, reason: events.append((frm, to, reason)) + + primary.generate_response(system_prompt="s", user_prompt="u") + assert events == [("openai", "anthropic", events[0][2])] diff --git a/tests/test_bedrock_token_normalization.py b/tests/test_bedrock_token_normalization.py index c0860885..58ce2a9d 100644 --- a/tests/test_bedrock_token_normalization.py +++ b/tests/test_bedrock_token_normalization.py @@ -52,6 +52,9 @@ def test_llm_bedrock_input_includes_cache_and_cached_is_reads_only(): iface = LLMInterface.__new__(LLMInterface) reported = _stub_common(iface) iface._call_log_to_db = lambda *a, **kw: None + # Read by the per-model reasoning lookup every transport performs. + iface._auth_mode = "api_key" + iface._reasoning_logged_for = None result = iface._generate_bedrock(None, "hi") From de5902c3f51d2d70552c7f8462cc4cff00bb267c Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 09:10:18 +0900 Subject: [PATCH 07/55] create effort setting knob --- agent_core/core/impl/action/context.py | 11 +- agent_core/core/impl/action/executor.py | 15 +- agent_core/core/impl/llm/interface.py | 72 +- agent_core/core/impl/llm/reasoning_wire.py | 86 +-- .../impl/llm/transports/bedrock_converse.py | 6 +- agent_core/core/impl/session/manager.py | 30 + agent_core/core/models/reasoning.py | 637 +++++++++++++----- agent_core/core/session/session.py | 17 + agent_core/core/state/session.py | 33 +- app/agent_base.py | 70 +- app/triggers/runtime.py | 6 +- app/ui_layer/adapters/browser_adapter.py | 83 ++- .../src/components/Chat/Chat.module.css | 6 + .../frontend/src/components/Chat/Chat.tsx | 78 ++- .../Chat/ReasoningPicker.module.css | 102 +++ .../src/components/Chat/ReasoningPicker.tsx | 159 +++++ .../frontend/src/components/layout/NavBar.tsx | 1 + .../src/contexts/WebSocketContext.tsx | 59 +- .../browser/frontend/src/locales/en/chat.json | 20 + .../browser/frontend/src/locales/es/chat.json | 20 + .../browser/frontend/src/locales/id/chat.json | 20 + .../browser/frontend/src/locales/ja/chat.json | 20 + .../browser/frontend/src/locales/ko/chat.json | 20 + .../frontend/src/locales/zh-CN/chat.json | 20 + .../frontend/src/locales/zh-TW/chat.json | 20 + .../browser/frontend/src/store/index.ts | 2 + .../frontend/src/store/resources/catalog.ts | 7 + .../frontend/src/store/selectors/reasoning.ts | 3 + .../src/store/slices/reasoningSlice.ts | 52 ++ .../frontend/src/store/uiState/catalog.ts | 9 +- .../browser/frontend/src/types/index.ts | 19 + app/ui_layer/controller/ui_controller.py | 14 +- scripts/probe_reasoning.py | 60 +- tests/llm/test_reasoning_rules.py | 632 ++++++++++++++--- 34 files changed, 2003 insertions(+), 406 deletions(-) create mode 100644 app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.module.css create mode 100644 app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.tsx create mode 100644 app/ui_layer/browser/frontend/src/store/selectors/reasoning.ts create mode 100644 app/ui_layer/browser/frontend/src/store/slices/reasoningSlice.ts diff --git a/agent_core/core/impl/action/context.py b/agent_core/core/impl/action/context.py index 66f0cde0..96039512 100644 --- a/agent_core/core/impl/action/context.py +++ b/agent_core/core/impl/action/context.py @@ -9,9 +9,9 @@ Scope rules: - Set only by the internal executors (``_atomic_action_internal*``), reset in a ``finally`` — never leaks across actions. - - Sync actions run in a thread pool where the caller's context does NOT - propagate, so the executor wraps the call and sets the var inside the - worker thread (see ``run_with_input_context``). + - Sync actions run in a thread pool inside a copy of the caller's + context; the executor wraps the call and sets the var inside the worker + thread (see ``run_with_input_context``). - Sandboxed (subprocess) actions cannot see it at all — helpers must treat a ``None`` value as "no context available". """ @@ -31,8 +31,9 @@ def run_with_input_context( ) -> dict: """Call a sync action with ``current_input_data`` set for its duration. - Used as the thread-pool target: the worker thread has its own context, - so the var must be set (and reset) inside the thread, not the caller. + Used as the thread-pool target, run inside a copy of the caller's + context: the var is set (and reset) inside the thread so it scopes to + this action alone. """ token = current_input_data.set(input_data) try: diff --git a/agent_core/core/impl/action/executor.py b/agent_core/core/impl/action/executor.py index 5b735dfd..6d15b734 100644 --- a/agent_core/core/impl/action/executor.py +++ b/agent_core/core/impl/action/executor.py @@ -14,6 +14,7 @@ """ import asyncio +import contextvars import importlib import json import os @@ -634,14 +635,20 @@ async def _atomic_action_internal_async( finally: current_input_data.reset(ctx_token) else: - # Sync function - run in thread pool to avoid blocking. The - # worker thread doesn't inherit this context, so the wrapper - # sets current_input_data inside the thread. + # Sync function - run in thread pool to avoid blocking. A pool + # thread does not inherit the caller's context, so the action + # runs inside a copy of it (as asyncio.to_thread does): the bound + # session, log context, and other context-scoped state follow the + # action into the thread, and the wrapper sets current_input_data + # there. logger.debug( f"[SYNC] Action '{action_name}' is sync, running in thread pool" ) thread_future = THREAD_POOL.submit( - run_with_input_context, function_to_call, input_data + contextvars.copy_context().run, + run_with_input_context, + function_to_call, + input_data, ) try: execution_result = await asyncio.wrap_future(thread_future) diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index 7a97fd51..520330cf 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -42,11 +42,19 @@ RecordLLMCallHook, ) from agent_core.core.impl.llm import transports as _transports -from agent_core.core.models.reasoning import ReasoningDecision, resolve_reasoning +from agent_core.core.models.reasoning import ( + ReasoningChoice, + ReasoningDecision, + ReasoningOptions, + default_choice, + reasoning_options as _reasoning_options, + resolve_reasoning, +) from agent_core.core.models.registry import ( get_registry as _get_registry, session_cc_providers as _session_cc_providers, ) +from agent_core.core.state.session import StateSession # Logging setup - use shared agent_core logger for consistency from agent_core.utils.logger import logger @@ -588,6 +596,7 @@ def _begin_call( call_type: Optional[str] = None, task_id: Optional[str] = None, thinking_budget: Optional[int] = None, + reasoning_choice: Optional[ReasoningChoice] = None, ) -> None: """Stamp per-call identity + start time into the context for capture. @@ -598,6 +607,9 @@ def _begin_call( ``thinking_budget`` (when set) is read by the Gemini transport to cap reasoning tokens; other transports never look at it. + + ``reasoning_choice`` (when set) overrides the bound session's choice + for this call; see ``reasoning_decision``. """ _llm_call_ctx.set( { @@ -605,6 +617,7 @@ def _begin_call( "call_type": call_type, "task_id": task_id, "thinking_budget": thinking_budget, + "reasoning_choice": reasoning_choice, "start": time.perf_counter(), } ) @@ -804,16 +817,52 @@ def _check_context_fits( f"{self.provider}/{self.model}." ) + def reasoning_options(self) -> Optional[ReasoningOptions]: + """What the chat-input reasoning picker offers for the model in use. + + None when the model has no rule (its reasoning is not adjustable). + """ + return _reasoning_options(self.provider, self.model, self._auth_mode) + + def default_reasoning_choice(self) -> ReasoningChoice: + """The choice a new session starts at with this interface's model.""" + return default_choice(self.provider, self.model, self._auth_mode) + + def reasoning_choice(self) -> ReasoningChoice: + """The reasoning choice that applies to the call in flight. + + A choice passed for this call wins (work for a chat that has no + session yet: the draft view); otherwise the choice of the session + the context is bound to (StateSession.bind, set around each + session's loop); otherwise, for work that belongs to no session, + the model's default level. + """ + explicit = _llm_call_ctx.get().get("reasoning_choice") + if explicit is not None: + return explicit + state = StateSession.bound() + if ( + state is not None + and state.current_session is not None + and state.current_session.reasoning_effort is not None + ): + return ReasoningChoice(state.current_session.reasoning_effort) + return self.default_reasoning_choice() + def reasoning_decision(self) -> Optional[ReasoningDecision]: - """Reasoning default for the current provider and model. + """What the call in flight sends for reasoning. None means the model has no rule in agent_core/core/models/reasoning.py and its requests carry no reasoning parameter. Resolved on every call - so a model switch (reinitialize) or a fallback interface uses its own - row; logged once per distinct model. + so a model switch (reinitialize), a fallback interface, or a changed + session choice applies from the next request; logged once per + distinct model and choice. """ - decision = resolve_reasoning(self.provider, self.model, self._auth_mode) - log_key = (self.provider, self.model, self._auth_mode) + choice = self.reasoning_choice() + decision = resolve_reasoning( + self.provider, self.model, self._auth_mode, choice + ) + log_key = (self.provider, self.model, self._auth_mode, choice) if log_key != self._reasoning_logged_for: self._reasoning_logged_for = log_key if decision is None: @@ -1008,6 +1057,7 @@ async def generate_response_async( prompt_name: Optional[str] = None, json_mode: bool = True, thinking_budget: Optional[int] = None, + reasoning_choice: Optional[ReasoningChoice] = None, ) -> str: """Async wrapper that defers the blocking call to a worker thread. @@ -1017,10 +1067,18 @@ async def generate_response_async( ``thinking_budget`` caps reasoning tokens on providers that expose a thinking budget (Gemini). It rides the per-call context and is a no-op for every other provider; leave it None (the default) for normal calls. + + ``reasoning_choice`` overrides the bound session's reasoning choice + for this call; pass it only for work done for a chat that has no + session yet. """ # Stamp the context here, in the caller's context, so asyncio.to_thread # copies it into the worker thread where the capture runs. - self._begin_call(prompt_name=prompt_name, thinking_budget=thinking_budget) + self._begin_call( + prompt_name=prompt_name, + thinking_budget=thinking_budget, + reasoning_choice=reasoning_choice, + ) return await asyncio.to_thread( self._generate_response_sync, system_prompt, diff --git a/agent_core/core/impl/llm/reasoning_wire.py b/agent_core/core/impl/llm/reasoning_wire.py index 9203801b..32d9189e 100644 --- a/agent_core/core/impl/llm/reasoning_wire.py +++ b/agent_core/core/impl/llm/reasoning_wire.py @@ -1,49 +1,36 @@ # -*- coding: utf-8 -*- -"""Render a reasoning default into request fields, one function per transport. +"""Render a reasoning decision into request fields, one function per transport. -The table and the level policy live in agent_core/core/models/reasoning.py; -this module only knows how each transport spells a ``ReasoningDecision``. -Callers skip these functions entirely when a model has no rule, so a model -without a rule never gains a field here. +The table, the choices, and the level policy live in +agent_core/core/models/reasoning.py; this module only knows how each +transport spells a ``ReasoningDecision``. Callers skip these functions +entirely when a model has no rule, so a model without a rule never gains a +field here. A decision that sends nothing (the provider default, or off on a +model whose default is no reasoning) renders as no fields. A decision whose wire the transport cannot express is a bug in the rules -table (a row filed under the wrong provider), so it raises instead of being -dropped silently. tests/llm/test_reasoning_rules.py checks every row against -``TRANSPORT_WIRES`` so this never reaches production. +table (a row filed under the wrong provider, or an explicit off on a wire +without a disable form), so it raises instead of being dropped silently. +tests/llm/test_reasoning_rules.py renders every choice on every row through +its provider's function, so this never reaches production. """ from __future__ import annotations -from typing import Any, Dict, FrozenSet, Mapping, Tuple +from typing import Any, Dict, Tuple from agent_core.core.models.reasoning import ReasoningDecision, ReasoningWire -#: Reasoning wires each transport (ProviderProfile.wire) can express. -TRANSPORT_WIRES: Mapping[str, FrozenSet[ReasoningWire]] = { - "chat_completions": frozenset( - { - ReasoningWire.EFFORT, - ReasoningWire.OPENROUTER_EFFORT, - ReasoningWire.OPENROUTER_BUDGET, - } - ), - "anthropic_messages": frozenset( - {ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET} - ), - "bedrock_converse": frozenset( - {ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET} - ), - "gemini_native": frozenset( - {ReasoningWire.GEMINI_LEVEL, ReasoningWire.GEMINI_BUDGET} - ), -} +#: The disable value of the OpenAI-style effort parameter. +EFFORT_OFF = "none" def _unsupported(decision: ReasoningDecision, transport: str) -> ValueError: + what = "an explicit off" if decision.off else f"wire {decision.wire.value!r}" return ValueError( - f"Reasoning rule {decision.key!r} uses wire {decision.wire.value!r}, " - f"which the {transport} transport cannot send. The row is filed under " - f"the wrong provider in agent_core/core/models/reasoning.py." + f"Reasoning rule {decision.key!r} asks for {what}, which the " + f"{transport} transport cannot send. Fix the row in " + f"agent_core/core/models/reasoning.py." ) @@ -52,10 +39,20 @@ def chat_completions_fields( ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """Fields for a Chat Completions request: (top-level kwargs, extra_body).""" if decision.wire is ReasoningWire.EFFORT: + if decision.off: + return {"reasoning_effort": EFFORT_OFF}, {} + if decision.level is None: + return {}, {} return {"reasoning_effort": decision.level}, {} if decision.wire is ReasoningWire.OPENROUTER_EFFORT: + if decision.off: + return {}, {"reasoning": {"effort": EFFORT_OFF}} + if decision.level is None: + return {}, {} return {}, {"reasoning": {"effort": decision.level}} - if decision.wire is ReasoningWire.OPENROUTER_BUDGET: + if decision.wire is ReasoningWire.OPENROUTER_BUDGET and not decision.off: + if decision.level is None: + return {}, {} return {}, {"reasoning": {"max_tokens": decision.budget_tokens}} raise _unsupported(decision, "chat_completions") @@ -66,22 +63,33 @@ def anthropic_fields(decision: ReasoningDecision) -> Dict[str, Any]: The same dict is the ``additionalModelRequestFields`` of a Bedrock Converse request for Claude, which forwards these keys to the model. """ + if decision.wire not in ( + ReasoningWire.ANTHROPIC_ADAPTIVE, + ReasoningWire.ANTHROPIC_BUDGET, + ): + raise _unsupported(decision, "anthropic_messages/bedrock_converse") + if decision.off: + return {"thinking": {"type": "disabled"}} + if decision.level is None: + return {} if decision.wire is ReasoningWire.ANTHROPIC_ADAPTIVE: return { "thinking": {"type": "adaptive"}, "output_config": {"effort": decision.level}, } - if decision.wire is ReasoningWire.ANTHROPIC_BUDGET: - return { - "thinking": {"type": "enabled", "budget_tokens": decision.budget_tokens} - } - raise _unsupported(decision, "anthropic_messages/bedrock_converse") + return {"thinking": {"type": "enabled", "budget_tokens": decision.budget_tokens}} def gemini_thinking_kwargs(decision: ReasoningDecision) -> Dict[str, Any]: """Keyword arguments for the GeminiClient text-generation methods.""" - if decision.wire is ReasoningWire.GEMINI_LEVEL: - return {"thinking_level": decision.level} if decision.wire is ReasoningWire.GEMINI_BUDGET: + if decision.off: + return {"thinking_budget": 0} + if decision.level is None: + return {} return {"thinking_budget": decision.budget_tokens} + if decision.wire is ReasoningWire.GEMINI_LEVEL and not decision.off: + if decision.level is None: + return {} + return {"thinking_level": decision.level} raise _unsupported(decision, "gemini_native") diff --git a/agent_core/core/impl/llm/transports/bedrock_converse.py b/agent_core/core/impl/llm/transports/bedrock_converse.py index bbf3abe2..7c41d07d 100644 --- a/agent_core/core/impl/llm/transports/bedrock_converse.py +++ b/agent_core/core/impl/llm/transports/bedrock_converse.py @@ -88,9 +88,9 @@ def generate( # through additionalModelRequestFields. Thinking tokens count # against maxTokens, and thinking is incompatible with (4.5/4.6) # or rejects (4.7+) a non-default temperature. - converse_kwargs["additionalModelRequestFields"] = ( - reasoning_wire.anthropic_fields(reasoning) - ) + reasoning_fields = reasoning_wire.anthropic_fields(reasoning) + if reasoning_fields: + converse_kwargs["additionalModelRequestFields"] = reasoning_fields converse_kwargs["inferenceConfig"]["maxTokens"] = reasoning.output_cap( iface.max_tokens ) diff --git a/agent_core/core/impl/session/manager.py b/agent_core/core/impl/session/manager.py index 35c12927..bd18bbdd 100644 --- a/agent_core/core/impl/session/manager.py +++ b/agent_core/core/impl/session/manager.py @@ -33,6 +33,7 @@ ) from agent_core.core.state import StateSession from agent_core.core.impl.llm import LLMCallType +from agent_core.core.models.reasoning import ReasoningChoice, default_choice from agent_core.utils.logger import logger @@ -137,6 +138,7 @@ def create_session( selected_skills: Optional[List[str]] = None, agent_app_project_id: Optional[str] = None, gui_mode: bool = False, + reasoning_effort: Optional[ReasoningChoice] = None, ) -> Session: """ Create a new persistent session. @@ -149,6 +151,9 @@ def create_session( selected_skills: Skills to preload (slash-command entry, Agent App). agent_app_project_id: Backing project for agent_app sessions. gui_mode: Whether the session starts in GUI mode. + reasoning_effort: The session's reasoning choice (the draft + chat's picker value when a draft becomes a session); None + starts it at the default level of the model in use. Returns: The created Session. @@ -179,6 +184,7 @@ def create_session( workspace_dir=str(workspace_dir), agent_app_project_id=agent_app_project_id, gui_mode=gui_mode, + reasoning_effort=(reasoning_effort or self._default_reasoning()).value, ) self.sessions[sid] = session @@ -275,6 +281,18 @@ def rename_session(self, session_id: str, title: str) -> bool: self._persist(session) return True + def set_reasoning_effort(self, session_id: str, choice: ReasoningChoice) -> bool: + """Set a session's reasoning choice (the chat input's picker). + + Takes effect from the session's next LLM request. + """ + session = self.sessions.get(session_id) + if not session: + return False + session.reasoning_effort = choice.value + self._persist(session) + return True + # ─────────────────────── Restore ───────────────────────────────────────── def restore_session(self, session: Session) -> Session: @@ -299,6 +317,11 @@ def restore_session(self, session: Session) -> Session: StateSession.start( session.id, current_session=session, gui_mode=session.gui_mode ) + if session.reasoning_effort is None: + # Saved without a valid reasoning choice: start it where a new + # session would. + session.reasoning_effort = self._default_reasoning().value + self._persist(session) return session # ─────────────────────── Todo Management ───────────────────────────────── @@ -542,6 +565,13 @@ def _create_session_caches(self, session_id: str) -> None: # ─────────────────────── Internal Helpers ──────────────────────────────── + def _default_reasoning(self) -> ReasoningChoice: + """The reasoning choice a session starts at: the default level of the + model in use.""" + if self.llm_interface is None: + return default_choice(None, None) + return self.llm_interface.default_reasoning_choice() + def _persist(self, session: Session) -> None: """Persist session state via hook.""" if self._on_session_persist: diff --git a/agent_core/core/models/reasoning.py b/agent_core/core/models/reasoning.py index 3f2301ce..8d6d5baf 100644 --- a/agent_core/core/models/reasoning.py +++ b/agent_core/core/models/reasoning.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Default reasoning effort ("thinking") per model. +"""Reasoning effort ("thinking") per model, chosen per chat session. Why this exists --------------- @@ -11,18 +11,34 @@ models of the SAME provider, and a value a model does not accept is a hard HTTP 400 on the first request. -So defaults live in a hard-coded table keyed by provider and EXACT model id -(no prefix or substring matching). A model that is not in the table gets no -reasoning parameter at all, and its request stays byte-identical to the one -sent before this module existed (pinned by the ``unruled_*`` golden payload -snapshots under tests/llm/golden/). - -Choosing the level ------------------- +So the capabilities live in a hard-coded table keyed by provider and EXACT +model id (no prefix or substring matching). A model that is not in the table +gets no reasoning parameter at all, whatever the session picked, and its +request stays byte-identical to the one sent before this module existed +(pinned by the ``unruled_*`` golden payload snapshots under tests/llm/golden/). + +Choices +------- +Each chat session stores a ``ReasoningChoice`` (``Session.reasoning_effort``, +picked in the chat input), following the pi agent harness: a concrete level, +``off``, or ``provider_default`` (no reasoning parameter, so the provider's +own default applies; on a backend that always needs a level, its documented +default level is sent). A new session starts at the model's default level +(see "The default level"), which the picker marks as "Default". + +A model offers its levels, plus ``off`` when it can stop reasoning. A stored +choice the current model lacks is clamped the way pi does it: the nearest +available choice at or above the request, else the nearest below, along +``CHOICE_LADDER``. The session keeps the REQUESTED choice, so it re-applies +after a model switch. + +The default level +----------------- ``ReasoningRule.levels`` lists the levels that turn reasoning on, weakest -first. "Off" values (``none``, ``disabled``, ``minimal``) and values a -provider silently maps onto another level are left out, so the tuple holds -only levels that behave differently. The default level is: +first. Values a provider silently maps onto another level are left out, so +the tuple holds only levels that behave differently. ``minimal`` is listed +where a model accepts it (it is selectable) but is never the default. The +default level is: 1. one level below the model's strongest level (``LEVELS_BELOW_MAX``); 2. never stronger than ``LEVEL_CEILING`` (``high``); @@ -31,15 +47,15 @@ level is used instead, so a default is never lowered. Token-budget models (Claude 4.5, Gemini 2.5) have no named levels; their -rows name four budget rungs ``BUDGET_RUNGS`` so the same rule picks the +rows name four budget rungs ``BUDGET_RUNGS`` so the same rules pick the third rung ("high"). Adding a model -------------- Add its exact id (every alias and dated snapshot the provider accepts) to a -rule below, with the ordered levels from the provider's documentation and -the documentation URL in ``source``. The unit tests in -tests/llm/test_reasoning_rules.py validate every row. +rule below, with the ordered levels, default, and off behavior from the +provider's documentation and the documentation URL in ``source``. The unit +tests in tests/llm/test_reasoning_rules.py validate every row. The keys follow the ``/`` shape of the model catalog planned in docs/PROVIDER_LAYER_CATCHUP.md section 10, so the rows can move @@ -52,17 +68,48 @@ from enum import Enum from typing import Dict, Mapping, Optional, Tuple +# ─────────────────────────────── choices ────────────────────────────── + + +class ReasoningChoice(str, Enum): + """What a chat session asks for (stored as ``Session.reasoning_effort``).""" + + PROVIDER_DEFAULT = "provider_default" + OFF = "off" + MINIMAL = "minimal" + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + XHIGH = "xhigh" + MAX = "max" + + +#: Clamping ladder, weakest first (pi's EXTENDED_THINKING_LEVELS). +CHOICE_LADDER: Tuple[ReasoningChoice, ...] = ( + ReasoningChoice.OFF, + ReasoningChoice.MINIMAL, + ReasoningChoice.LOW, + ReasoningChoice.MEDIUM, + ReasoningChoice.HIGH, + ReasoningChoice.XHIGH, + ReasoningChoice.MAX, +) + # ─────────────────────────────── policy ─────────────────────────────── #: How many levels below the model's strongest level the default sits. LEVELS_BELOW_MAX = 1 -#: The strongest level ever chosen as a default (unless the provider's own -#: default is already stronger; see rule 3 in the module docstring). +#: The strongest default level (unless the provider's own default is already +#: stronger; see rule 3 in the module docstring). It is also where a session +#: starts when it is created while the model in use has no rule. LEVEL_CEILING = "high" -#: Every level name any provider uses, weakest first. Rows must use these. -KNOWN_LEVELS: Tuple[str, ...] = ("minimal", "low", "medium", "high", "xhigh", "max") +#: Levels that are never the default: "minimal" barely reasons. +DEFAULT_EXCLUDED_LEVELS = frozenset({"minimal"}) + +#: Levels whose thinking needs the rule's extended output cap. +EXTENDED_LEVELS = frozenset({"xhigh", "max"}) #: Rung names for token-budget models, weakest first. BUDGET_RUNGS: Tuple[str, ...] = ("low", "medium", "high", "max") @@ -73,22 +120,25 @@ class ReasoningWire(str, Enum): - """How a reasoning default is expressed in a request.""" + """How reasoning is expressed in a request.""" - #: Top-level ``reasoning_effort: `` on a Chat Completions request. - #: The ChatGPT-subscription translator maps it to ``reasoning.effort``. + #: Top-level ``reasoning_effort: `` on a Chat Completions request + #: (off: ``"none"``). The ChatGPT-subscription translator maps it to + #: ``reasoning.effort``. EFFORT = "effort" #: Anthropic ``thinking: {"type": "adaptive"}`` plus - #: ``output_config: {"effort": }`` (Claude 4.6 and later). + #: ``output_config: {"effort": }`` (Claude 4.6 and later; off: + #: ``thinking: {"type": "disabled"}``). ANTHROPIC_ADAPTIVE = "anthropic_adaptive" #: Anthropic ``thinking: {"type": "enabled", "budget_tokens": }`` #: (Claude 4.5 generation). ANTHROPIC_BUDGET = "anthropic_budget" #: Gemini ``thinkingConfig: {"thinkingLevel": }`` (Gemini 3.x). GEMINI_LEVEL = "gemini_level" - #: Gemini ``thinkingConfig: {"thinkingBudget": }`` (Gemini 2.5). + #: Gemini ``thinkingConfig: {"thinkingBudget": }`` (Gemini 2.5; off: + #: budget 0). GEMINI_BUDGET = "gemini_budget" - #: OpenRouter ``reasoning: {"effort": }``. + #: OpenRouter ``reasoning: {"effort": }`` (off: ``"none"``). OPENROUTER_EFFORT = "openrouter_effort" #: OpenRouter ``reasoning: {"max_tokens": }``. OPENROUTER_BUDGET = "openrouter_budget" @@ -104,41 +154,92 @@ class ReasoningWire(str, Enum): ) +class ReasoningOff(str, Enum): + """How a model stops reasoning (a rule without one cannot).""" + + #: The provider's default is no reasoning: off sends nothing. + OMIT = "omit" + #: Off sends the wire's explicit disable form. + EXPLICIT = "explicit" + + +class TemperaturePolicy(str, Enum): + """When a model rejects an explicit ``temperature``.""" + + KEEP = "keep" + OMIT_WHILE_THINKING = "omit_while_thinking" + OMIT_ALWAYS = "omit_always" + + @dataclass(frozen=True) class ReasoningRule: """Reasoning capabilities of one model (or of models that share them).""" - #: How the default is expressed in the request. + #: How reasoning is expressed in the request. wire: ReasoningWire #: Levels that turn reasoning on, weakest first (``BUDGET_RUNGS`` for #: budget wires). levels: Tuple[str, ...] #: Level the provider applies when the parameter is omitted, when that - #: is one of ``levels``. None when omitting it means no reasoning, a - #: provider-chosen dynamic amount, or an "off" value. + #: is one of ``levels``. None when omitting it means no reasoning or a + #: provider-chosen (dynamic or undocumented) amount. provider_default: Optional[str] #: Documentation the row was taken from. source: str #: Budget-wire rows: token budget per rung, aligned with ``levels``. budgets: Tuple[int, ...] = () - #: Output-token cap to request while reasoning is on. Reasoning tokens - #: count against the provider's output cap, so it must leave room for - #: the answer. 0 keeps the caller's cap unchanged. + #: Output-token cap to request while reasoning. Reasoning tokens count + #: against the provider's output cap, so it must leave room for the + #: answer. 0 keeps the caller's cap unchanged. output_tokens: int = 0 - #: The model rejects ``temperature`` while reasoning is on. - omit_temperature: bool = False + #: Output-token cap at ``EXTENDED_LEVELS``; 0 means ``output_tokens``. + extended_output_tokens: int = 0 + #: How the model stops reasoning; None when it cannot. + off: Optional[ReasoningOff] = None + #: When the model rejects an explicit temperature. + temperature: TemperaturePolicy = TemperaturePolicy.KEEP + #: The backend requires a level on every request (the Codex backend), so + #: ``provider_default`` sends the documented default level explicitly. + requires_level: bool = False + + def ladder_choices(self) -> Tuple[ReasoningChoice, ...]: + """The ladder choices this model accepts, weakest first.""" + return tuple( + choice + for choice in CHOICE_LADDER + if (choice is ReasoningChoice.OFF and self.off is not None) + or choice.value in self.levels + ) + + def output_tokens_for(self, level: Optional[str]) -> int: + """Output cap while reasoning at ``level`` (None: the provider's).""" + if level in EXTENDED_LEVELS and self.extended_output_tokens: + return self.extended_output_tokens + return self.output_tokens + + def budget_for(self, level: str) -> int: + """Token budget of a budget-wire rung.""" + return self.budgets[self.levels.index(level)] @dataclass(frozen=True) class ReasoningDecision: - """The reasoning default resolved for one (provider, model) pair.""" + """What one request sends, resolved from a model's rule and a choice.""" #: ``/`` of the row that produced this decision. key: str + #: The choice the session asked for. + requested: ReasoningChoice + #: The choice in effect after clamping it to this model. + choice: ReasoningChoice wire: ReasoningWire - level: str + #: Level sent, or None when no level is sent (off, provider default). + level: Optional[str] #: Token budget for budget wires, else None. budget_tokens: Optional[int] + #: Send the wire's explicit disable form. + off: bool + #: Output cap the request needs; 0 keeps the caller's cap. output_tokens: int omit_temperature: bool @@ -151,20 +252,69 @@ def output_cap(self, base: int) -> int: def describe(self) -> str: """One-line human-readable summary for logs.""" - if self.budget_tokens is not None: - return f"{self.wire.value} budget={self.budget_tokens} ({self.level})" - return f"{self.wire.value} level={self.level}" + choice = self.requested.value + if self.choice is not self.requested: + choice = f"{choice} (clamped to {self.choice.value})" + if self.off: + sent = "disable" + elif self.level is None and self.choice is ReasoningChoice.OFF: + sent = "nothing (the model does not reason by default)" + elif self.level is None: + sent = "nothing (provider default)" + elif self.budget_tokens is not None: + sent = f"budget={self.budget_tokens} ({self.level})" + else: + sent = f"level={self.level}" + return f"choice={choice}, {self.wire.value} {sent}" + + +@dataclass(frozen=True) +class ReasoningOptions: + """What the chat-input picker offers for one model.""" + + #: Selectable choices, in display order. + choices: Tuple[ReasoningChoice, ...] + #: The model's default level (marked "Default" in the picker). + default_level: str + #: What ``provider_default`` does: "off", a level, or "dynamic". + provider_default: str + #: Effective choice for every possible stored choice. + resolution: Mapping[ReasoningChoice, ReasoningChoice] + + +def default_level(rule: ReasoningRule) -> str: + """The default level for ``rule`` (see the module docstring).""" + ladder = tuple( + level for level in rule.levels if level not in DEFAULT_EXCLUDED_LEVELS + ) + index = max(len(ladder) - 1 - LEVELS_BELOW_MAX, 0) + if LEVEL_CEILING in ladder: + index = min(index, ladder.index(LEVEL_CEILING)) + if rule.provider_default in ladder and index < ladder.index(rule.provider_default): + index = len(ladder) - 1 + return ladder[index] -def target_level(rule: ReasoningRule) -> str: - """Pick the default level for ``rule`` (see the module docstring).""" - levels = rule.levels - index = max(len(levels) - 1 - LEVELS_BELOW_MAX, 0) - if LEVEL_CEILING in levels: - index = min(index, levels.index(LEVEL_CEILING)) - if rule.provider_default in levels and index < levels.index(rule.provider_default): - index = len(levels) - 1 - return levels[index] +def clamp_choice(rule: ReasoningRule, choice: ReasoningChoice) -> ReasoningChoice: + """The choice that takes effect for ``rule`` when a session asks for ``choice``. + + Ladder choices the model lacks move to the nearest available choice at or + above the request, else the nearest below (pi's ``clampThinkingLevel``). + """ + if choice is ReasoningChoice.PROVIDER_DEFAULT: + # Omitting the parameter IS off on these models. + return ReasoningChoice.OFF if rule.off is ReasoningOff.OMIT else choice + available = rule.ladder_choices() + if choice in available: + return choice + requested = CHOICE_LADDER.index(choice) + for candidate in CHOICE_LADDER[requested:]: + if candidate in available: + return candidate + for candidate in reversed(CHOICE_LADDER[:requested]): + if candidate in available: + return candidate + raise ValueError(f"Reasoning rule has no levels: {rule!r}") def reasoning_surface(provider: str, auth_mode: str) -> str: @@ -178,29 +328,93 @@ def reasoning_surface(provider: str, auth_mode: str) -> str: return provider +def _lookup( + provider: Optional[str], model: Optional[str], auth_mode: str +) -> Tuple[Optional[str], Optional[ReasoningRule]]: + if not provider or not model: + return None, None + surface = reasoning_surface(provider, auth_mode) + return f"{surface}/{model}", REASONING_RULES.get(surface, {}).get(model) + + +def default_choice( + provider: Optional[str], model: Optional[str], auth_mode: str = "api_key" +) -> ReasoningChoice: + """The choice a new session starts at with this model in use. + + The model's default level; ``LEVEL_CEILING`` when the model has no rule + (the session then takes a level that most models offer, clamped to the + next model with a rule). + """ + _, rule = _lookup(provider, model, auth_mode) + return ReasoningChoice(default_level(rule) if rule is not None else LEVEL_CEILING) + + def resolve_reasoning( provider: Optional[str], model: Optional[str], - auth_mode: str = "api_key", + auth_mode: str, + choice: ReasoningChoice, ) -> Optional[ReasoningDecision]: - """Resolve the reasoning default for a model, or None if it has no row.""" - if not provider or not model: - return None - surface = reasoning_surface(provider, auth_mode) - rule = REASONING_RULES.get(surface, {}).get(model) + """Resolve what a request sends for ``choice``, or None if the model has + no row.""" + key, rule = _lookup(provider, model, auth_mode) if rule is None: return None - level = target_level(rule) - budget = ( - rule.budgets[rule.levels.index(level)] if rule.wire in BUDGET_WIRES else None - ) + effective = clamp_choice(rule, choice) + if effective is ReasoningChoice.PROVIDER_DEFAULT: + level = rule.provider_default if rule.requires_level else None + elif effective is ReasoningChoice.OFF: + level = None + else: + level = effective.value + + if level is not None: + thinking = True + elif effective is ReasoningChoice.PROVIDER_DEFAULT: + thinking = rule.off is not ReasoningOff.OMIT + else: + thinking = False + return ReasoningDecision( - key=f"{surface}/{model}", + key=key, + requested=choice, + choice=effective, wire=rule.wire, level=level, - budget_tokens=budget, - output_tokens=rule.output_tokens, - omit_temperature=rule.omit_temperature, + budget_tokens=( + rule.budget_for(level) + if level is not None and rule.wire in BUDGET_WIRES + else None + ), + off=effective is ReasoningChoice.OFF and rule.off is ReasoningOff.EXPLICIT, + output_tokens=rule.output_tokens_for(level) if thinking else 0, + omit_temperature=( + rule.temperature is TemperaturePolicy.OMIT_ALWAYS + or (rule.temperature is TemperaturePolicy.OMIT_WHILE_THINKING and thinking) + ), + ) + + +def reasoning_options( + provider: Optional[str], model: Optional[str], auth_mode: str +) -> Optional[ReasoningOptions]: + """What the picker offers for a model, or None if the model has no row.""" + _, rule = _lookup(provider, model, auth_mode) + if rule is None: + return None + if rule.off is ReasoningOff.OMIT: + # Omitting the parameter is off: provider default and off coincide. + meta: Tuple[ReasoningChoice, ...] = () + provider_default = ReasoningChoice.OFF.value + else: + meta = (ReasoningChoice.PROVIDER_DEFAULT,) + provider_default = rule.provider_default or "dynamic" + return ReasoningOptions( + choices=meta + rule.ladder_choices(), + default_level=default_level(rule), + provider_default=provider_default, + resolution={choice: clamp_choice(rule, choice) for choice in ReasoningChoice}, ) @@ -240,28 +454,34 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: # Output caps. Non-streaming Anthropic requests are refused by the Anthropic # SDK above 21,333 max_tokens (anthropic/_base_client.py # _calculate_nonstreaming_timeout), so Anthropic-family rows stay just below -# it. OpenAI-style effort rows use 32,000: OpenAI recommends reserving at -# least 25,000 tokens for reasoning plus output -# (https://developers.openai.com/api/docs/guides/reasoning). +# it at every level. OpenAI-style effort rows use 32,000 (OpenAI recommends +# reserving at least 25,000 tokens for reasoning plus output, +# https://developers.openai.com/api/docs/guides/reasoning) and 64,000 at +# xhigh/max, where thinking alone can pass 32,000; every such model's +# documented output limit is above that. _ANTHROPIC_OUTPUT_TOKENS = 21_000 _EFFORT_OUTPUT_TOKENS = 32_000 +_EFFORT_EXTENDED_OUTPUT_TOKENS = 64_000 # ── OpenAI public API (Chat Completions ``reasoning_effort``) ── # Values and defaults: https://developers.openai.com/api/docs/models/. -# Models before gpt-5.1 reject "none"; gpt-5.1+ reject "minimal"; xhigh -# arrived with gpt-5.2 and max with gpt-5.6. Temperature is rejected whenever -# effort is not "none" (the OpenAI profile already omits it). +# Models before gpt-5.1 reject "none" and never accept temperature; gpt-5.1+ +# reject "minimal", default to no reasoning through gpt-5.4, and accept +# temperature only at "none". xhigh arrived with gpt-5.2. "max" exists only on +# the Responses API: Chat Completions rejects it on gpt-5.6 and GPT-6 ("Supported +# values are: 'none', 'low', 'medium', 'high', and 'xhigh'", live API error, +# 2026-10-02; GPT-6 Astra and 6.1 Sol list the same without 'none'). _OPENAI_DOCS = "https://developers.openai.com/api/docs/models" _OPENAI_GPT5 = ReasoningRule( wire=ReasoningWire.EFFORT, - levels=("low", "medium", "high"), + levels=("minimal", "low", "medium", "high"), provider_default="medium", source=f"{_OPENAI_DOCS}/gpt-5", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + temperature=TemperaturePolicy.OMIT_ALWAYS, ) _OPENAI_O_SERIES = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -269,23 +489,26 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="medium", source="https://developers.openai.com/api/docs/guides/reasoning", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + temperature=TemperaturePolicy.OMIT_ALWAYS, ) _OPENAI_GPT51 = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high"), - provider_default=None, # "none": no reasoning + provider_default=None, source=f"{_OPENAI_DOCS}/gpt-5.1", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT52_TO_54 = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high", "xhigh"), - provider_default=None, # "none": no reasoning + provider_default=None, source=f"{_OPENAI_DOCS}/gpt-5.2", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT55 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -293,30 +516,41 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="medium", source=f"{_OPENAI_DOCS}/gpt-5.5", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT56_PLUS = ReasoningRule( wire=ReasoningWire.EFFORT, - levels=("low", "medium", "high", "xhigh", "max"), + levels=("low", "medium", "high", "xhigh"), provider_default="medium", source=f"{_OPENAI_DOCS}/gpt-6-sol", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) -#: GPT-6 Astra rejects "none" and documents no default. -_OPENAI_GPT6_ASTRA = ReasoningRule( +#: GPT-6 Astra and GPT-6.1 Sol reject "none" and always reason. +_OPENAI_GPT6_ALWAYS_REASONING = ReasoningRule( wire=ReasoningWire.EFFORT, - levels=("low", "medium", "high", "xhigh", "max"), + levels=("low", "medium", "high", "xhigh"), provider_default=None, source=f"{_OPENAI_DOCS}/gpt-6-astra", output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + temperature=TemperaturePolicy.OMIT_ALWAYS, +) +_OPENAI_GPT61_SOL = replace( + _OPENAI_GPT6_ALWAYS_REASONING, + provider_default="medium", + source=f"{_OPENAI_DOCS}/gpt-6.1-sol", ) # ── ChatGPT subscription (Codex backend ``reasoning.effort``) ── -# The translator always sends a reasoning block and drops output caps, so -# output_tokens stays 0. Levels are the catalogue's supported_reasoning_levels -# minus the client-side "ultra" alias +# The translator always sends a reasoning block and drops output caps and +# temperature, so output_tokens stays 0. Levels are the catalogue's +# supported_reasoning_levels minus the client-side "ultra" alias; no model +# lists "none", so none can be turned off # (https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json). _CODEX_CATALOG = ( @@ -327,27 +561,27 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: levels=("low", "medium", "high", "xhigh"), provider_default="medium", source=_CODEX_CATALOG, + requires_level=True, ) _CODEX_MAX_LADDER_LOW_DEFAULT = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high", "xhigh", "max"), provider_default="low", source=_CODEX_CATALOG, + requires_level=True, ) -_CODEX_MAX_LADDER_MEDIUM_DEFAULT = ReasoningRule( - wire=ReasoningWire.EFFORT, - levels=("low", "medium", "high", "xhigh", "max"), - provider_default="medium", - source=_CODEX_CATALOG, +_CODEX_MAX_LADDER_MEDIUM_DEFAULT = replace( + _CODEX_MAX_LADDER_LOW_DEFAULT, provider_default="medium" ) # ── OpenRouter (``reasoning`` object) ── # Levels: live GET https://openrouter.ai/api/v1/models ``reasoning`` -# (supported_efforts). Claude 4.5 slugs expose no effort selector, only a +# (supported_efforts; "mandatory" models cannot be turned off, the others +# take effort "none"). Claude 4.5 slugs expose no effort selector, only a # token budget, which must be >= 1024 and strictly below the request # max_tokens (https://openrouter.ai/docs/guides/best-practices/reasoning-tokens). -# Explicitly sent sampling parameters are forwarded upstream, where thinking -# rejects them, so every OpenRouter row drops temperature. +# Upstream Claude does not think unless asked. Explicitly sent sampling +# parameters are forwarded upstream, where thinking rejects them. _OPENROUTER_MODELS = "https://openrouter.ai/api/v1/models" _OPENROUTER_CLAUDE_BUDGET = ReasoningRule( @@ -357,115 +591,135 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: source="https://openrouter.ai/docs/guides/best-practices/reasoning-tokens", budgets=(4_096, 8_192, 12_288, 16_384), output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) -_OPENROUTER_CLAUDE_SONNET_46 = ReasoningRule( +_OPENROUTER_CLAUDE_46 = ReasoningRule( wire=ReasoningWire.OPENROUTER_EFFORT, levels=("low", "medium", "high", "max"), - provider_default="medium", + provider_default=None, source=_OPENROUTER_MODELS, output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, -) -_OPENROUTER_CLAUDE_OPUS_46 = replace( - _OPENROUTER_CLAUDE_SONNET_46, provider_default="high" + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENROUTER_OPENAI_GPT5 = ReasoningRule( wire=ReasoningWire.OPENROUTER_EFFORT, - levels=("low", "medium", "high"), + levels=("minimal", "low", "medium", "high"), provider_default="medium", source=_OPENROUTER_MODELS, output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + temperature=TemperaturePolicy.OMIT_ALWAYS, ) -_OPENROUTER_OPENAI_GPT51 = replace(_OPENROUTER_OPENAI_GPT5, provider_default=None) -_OPENROUTER_OPENAI_XHIGH = ReasoningRule( +_OPENROUTER_OPENAI_GPT51 = ReasoningRule( wire=ReasoningWire.OPENROUTER_EFFORT, - levels=("low", "medium", "high", "xhigh"), - provider_default="medium", + levels=("low", "medium", "high"), + provider_default=None, source=_OPENROUTER_MODELS, output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) -_OPENROUTER_OPENAI_MAX = ReasoningRule( +_OPENROUTER_OPENAI_XHIGH = ReasoningRule( wire=ReasoningWire.OPENROUTER_EFFORT, - levels=("low", "medium", "high", "xhigh", "max"), + levels=("low", "medium", "high", "xhigh"), provider_default="medium", source=_OPENROUTER_MODELS, output_tokens=_EFFORT_OUTPUT_TOKENS, - omit_temperature=True, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, +) +_OPENROUTER_OPENAI_MAX = replace( + _OPENROUTER_OPENAI_XHIGH, levels=("low", "medium", "high", "xhigh", "max") +) +_OPENROUTER_OPENAI_MAX_MANDATORY = replace( + _OPENROUTER_OPENAI_MAX, off=None, temperature=TemperaturePolicy.OMIT_ALWAYS ) # ── Google Gemini (generateContent ``thinkingConfig``) ── # https://ai.google.dev/gemini-api/docs/generate-content/thinking. 2.5 models -# take a token budget only (thinkingLevel is a 400 on them); 3.x models take -# a lowercase thinkingLevel. Thinking tokens count toward maxOutputTokens -# (65,536 on every model below), so the cap leaves 8,192 tokens of answer on -# top of the chosen budget. "minimal" is excluded: it is near-off, and an -# error on 3.1-pro / 3.7-flash / 3.8-flash. +# take a token budget only (thinkingLevel is a 400 on them); flash and +# flash-lite can turn thinking off with budget 0, pro cannot. 3.x models take +# a lowercase thinkingLevel and cannot turn thinking off; "minimal" is an +# error on 3.1-pro, 3.7-flash, and 3.8-flash. Thinking tokens count toward +# maxOutputTokens (65,536 on every model below), so each cap leaves 8,192 +# tokens of answer on top of the largest budget it covers. _GEMINI_THINKING_DOCS = ( "https://ai.google.dev/gemini-api/docs/generate-content/thinking" ) -_GEMINI_OUTPUT_TOKENS = 32_768 +_GEMINI_ANSWER_TOKENS = 8_192 _GEMINI_25_PRO = ReasoningRule( wire=ReasoningWire.GEMINI_BUDGET, levels=BUDGET_RUNGS, provider_default=None, # dynamic thinking source=_GEMINI_THINKING_DOCS, budgets=(8_192, 16_384, 24_576, 32_768), - output_tokens=24_576 + 8_192, + output_tokens=24_576 + _GEMINI_ANSWER_TOKENS, + extended_output_tokens=32_768 + _GEMINI_ANSWER_TOKENS, ) _GEMINI_25_FLASH = ReasoningRule( wire=ReasoningWire.GEMINI_BUDGET, levels=BUDGET_RUNGS, - provider_default=None, # dynamic (flash) or no thinking (flash-lite) + provider_default=None, # dynamic thinking source=_GEMINI_THINKING_DOCS, budgets=(6_144, 12_288, 18_432, 24_576), - output_tokens=18_432 + 8_192, + output_tokens=18_432 + _GEMINI_ANSWER_TOKENS, + extended_output_tokens=24_576 + _GEMINI_ANSWER_TOKENS, + off=ReasoningOff.EXPLICIT, ) -_GEMINI_3_HIGH_DEFAULT = ReasoningRule( +#: flash-lite does not think unless given a budget. +_GEMINI_25_FLASH_LITE = replace(_GEMINI_25_FLASH, off=ReasoningOff.OMIT) +_GEMINI_3_OUTPUT_TOKENS = 32_768 +_GEMINI_3_WITH_MINIMAL = ReasoningRule( wire=ReasoningWire.GEMINI_LEVEL, - levels=("low", "medium", "high"), - provider_default="high", + levels=("minimal", "low", "medium", "high"), + provider_default="medium", source=_GEMINI_THINKING_DOCS, - output_tokens=_GEMINI_OUTPUT_TOKENS, + output_tokens=_GEMINI_3_OUTPUT_TOKENS, +) +_GEMINI_3_WITHOUT_MINIMAL = replace( + _GEMINI_3_WITH_MINIMAL, levels=("low", "medium", "high") ) -_GEMINI_3_MEDIUM_DEFAULT = replace(_GEMINI_3_HIGH_DEFAULT, provider_default="medium") -_GEMINI_3_MINIMAL_DEFAULT = replace(_GEMINI_3_HIGH_DEFAULT, provider_default=None) # ── xAI (Chat Completions ``reasoning_effort``) ── -# https://docs.x.ai/developers/models/. No Grok model accepts "max", and -# "none" is documented only for grok-4.3. grok-4.5 treats "xhigh" as "high", -# so its distinct levels stop at high. +# https://docs.x.ai/developers/models/. No Grok model accepts "max"; +# "none" (off) is documented only for grok-4.3. grok-4.5 treats "xhigh" as +# "high", so its distinct levels stop at high. _XAI_DOCS = "https://docs.x.ai/developers/models" +_XAI_REASONING_DOCS = "https://docs.x.ai/developers/model-capabilities/text/reasoning" _XAI_GROK_43 = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high", "xhigh"), provider_default="low", source=f"{_XAI_DOCS}/grok-4.3", output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, ) _XAI_GROK_45 = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high"), provider_default="high", - source="https://docs.x.ai/developers/model-capabilities/text/reasoning", + source=_XAI_REASONING_DOCS, output_tokens=_EFFORT_OUTPUT_TOKENS, ) _XAI_GROK_46_PLUS = ReasoningRule( wire=ReasoningWire.EFFORT, levels=("low", "medium", "high", "xhigh"), provider_default="high", - source="https://docs.x.ai/developers/model-capabilities/text/reasoning", + source=_XAI_REASONING_DOCS, output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, ) # ── DeepSeek (``reasoning_effort``) ── # https://api-docs.deepseek.com/api/create-chat-completion: none/low/high/max -# (minimal maps to low, medium/xhigh to high), default high; max_tokens up to -# 393,216. Thinking mode ignores temperature without an error. +# (minimal maps to low, medium/xhigh to high), default high, "none" disables +# thinking; max_tokens up to 393,216. Thinking mode ignores temperature +# without an error. _DEEPSEEK_V4 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -473,12 +727,15 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="high", source="https://api-docs.deepseek.com/api/create-chat-completion", output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, ) # ── Z.ai GLM (``reasoning_effort``, standard route) ── # https://docs.z.ai/api-reference/llm/chat-completion, default max, 128K -# output. glm-5.3 accepts only low/high/max; glm-5.2 maps low/medium to high -# and xhigh to max, so its distinct levels are high and max. +# output. glm-5.3 accepts only low/high/max and cannot stop thinking; +# glm-5.2 maps low/medium to high and xhigh to max (distinct levels: high and +# max), and "none" skips thinking. _GLM_DOCS = "https://docs.z.ai/api-reference/llm/chat-completion" _GLM_53 = ReasoningRule( @@ -487,6 +744,7 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="max", source=_GLM_DOCS, output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, ) _GLM_52 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -494,6 +752,8 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="max", source=_GLM_DOCS, output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, + off=ReasoningOff.EXPLICIT, ) # ── Moonshot Kimi (``reasoning_effort``) ── @@ -506,11 +766,13 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="max", source="https://platform.kimi.ai/docs/guide/kimi-k3-quickstart", output_tokens=_EFFORT_OUTPUT_TOKENS, + extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, ) # ── gpt-oss on Groq / Cerebras (``reasoning_effort``) ── -# low/medium/high, default medium; out-of-set values are a 400. Output caps -# are clamped further by each provider profile's max_output_tokens. +# low/medium/high, default medium; always reasons; out-of-set values are a +# 400. Output caps are clamped further by each provider profile's +# max_output_tokens. _GPT_OSS_GROQ = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -523,43 +785,45 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: _GPT_OSS_GROQ, source="https://inference-docs.cerebras.ai/capabilities/reasoning" ) +# ── Anthropic Claude (Messages API) ── +# https://platform.claude.com/docs/en/build-with-claude/thinking-troubleshooting: +# Claude 4.5/4.6/4.7/4.8 do not think unless asked; Sonnet 5 and Opus 5 think +# by default and accept {"type": "disabled"}; Sonnet 5.5, Opus 5.5, Fable, +# and Mythos always think. Temperature is incompatible with thinking on 4.5 +# and 4.6 and rejected outright from 4.7 on. + _CLAUDE_DOCS = "https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking" _CLAUDE_BUDGET_DOCS = ( "https://platform.claude.com/docs/en/build-with-claude/extended-thinking" ) -#: Claude 4.6: adaptive thinking is off unless requested; effort has no -#: xhigh; sampling parameters are incompatible with thinking. +#: Claude 4.6: adaptive thinking off unless requested; effort has no xhigh. _CLAUDE_46 = ReasoningRule( wire=ReasoningWire.ANTHROPIC_ADAPTIVE, levels=("low", "medium", "high", "max"), - provider_default="high", - source=_CLAUDE_DOCS, - output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, -) - -#: Claude 4.7 and later: adaptive is the only thinking mode, the full effort -#: ladder exists, and non-default sampling parameters are a 400. -_CLAUDE_47_PLUS = ReasoningRule( - wire=ReasoningWire.ANTHROPIC_ADAPTIVE, - levels=("low", "medium", "high", "xhigh", "max"), - provider_default="high", + provider_default=None, source=_CLAUDE_DOCS, output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) - -#: Claude Opus 5.5: same surface, but its effort default is medium. -_CLAUDE_OPUS_55 = ReasoningRule( +#: Claude Opus 4.7 / 4.8: adaptive only, off unless requested. +_CLAUDE_47_48 = ReasoningRule( wire=ReasoningWire.ANTHROPIC_ADAPTIVE, levels=("low", "medium", "high", "xhigh", "max"), - provider_default="medium", + provider_default=None, source=_CLAUDE_DOCS, output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_ALWAYS, ) - +#: Claude Sonnet 5 / Opus 5: adaptive by default at effort high; disabling +#: is accepted at effort high or below (no effort is sent with it). +_CLAUDE_5 = replace(_CLAUDE_47_48, provider_default="high", off=ReasoningOff.EXPLICIT) +#: Claude Sonnet 5.5, Fable, Mythos: always think at effort high by default. +_CLAUDE_ALWAYS_THINKING = replace(_CLAUDE_47_48, provider_default="high", off=None) +#: Claude Opus 5.5: always thinks; its effort default is medium. +_CLAUDE_OPUS_55 = replace(_CLAUDE_ALWAYS_THINKING, provider_default="medium") #: Claude 4.5 generation: manual budget thinking only (adaptive is a 400, #: and effort is a 400 on Sonnet 4.5 / Haiku 4.5, so it is never sent). #: budget_tokens must be >= 1024 and < max_tokens. @@ -570,10 +834,10 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: source=_CLAUDE_BUDGET_DOCS, budgets=(4_096, 8_192, 12_288, 16_384), output_tokens=_ANTHROPIC_OUTPUT_TOKENS, - omit_temperature=True, + off=ReasoningOff.OMIT, + temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) - #: Claude on AWS Bedrock Converse takes the same thinking/effort keys through #: additionalModelRequestFields, with the same per-model rules #: (https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-adaptive-thinking.html, @@ -584,7 +848,9 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: "claude-messages-adaptive-thinking.html" ) _BEDROCK_CLAUDE_46 = replace(_CLAUDE_46, source=_BEDROCK_DOCS) -_BEDROCK_CLAUDE_47_PLUS = replace(_CLAUDE_47_PLUS, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_47_48 = replace(_CLAUDE_47_48, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_5 = replace(_CLAUDE_5, source=_BEDROCK_DOCS) +_BEDROCK_CLAUDE_ALWAYS_THINKING = replace(_CLAUDE_ALWAYS_THINKING, source=_BEDROCK_DOCS) _BEDROCK_CLAUDE_OPUS_55 = replace(_CLAUDE_OPUS_55, source=_BEDROCK_DOCS) _BEDROCK_CLAUDE_45_BUDGET = replace( _CLAUDE_45_BUDGET, @@ -636,9 +902,9 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna", - "gpt-6.1-sol", ), - _models(_OPENAI_GPT6_ASTRA, "gpt-6-astra"), + _models(_OPENAI_GPT6_ALWAYS_REASONING, "gpt-6-astra"), + _models(_OPENAI_GPT61_SOL, "gpt-6.1-sol"), ), "openai_subscription": _surface( _models(_CODEX_GPT55, "gpt-5.5"), @@ -660,8 +926,11 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: "anthropic/claude-haiku-4.5", "anthropic/claude-opus-4.5", ), - _models(_OPENROUTER_CLAUDE_SONNET_46, "anthropic/claude-sonnet-4.6"), - _models(_OPENROUTER_CLAUDE_OPUS_46, "anthropic/claude-opus-4.6"), + _models( + _OPENROUTER_CLAUDE_46, + "anthropic/claude-sonnet-4.6", + "anthropic/claude-opus-4.6", + ), _models(_OPENROUTER_OPENAI_GPT5, "openai/gpt-5", "openai/gpt-5-mini"), _models(_OPENROUTER_OPENAI_GPT51, "openai/gpt-5.1"), _models( @@ -677,28 +946,32 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: "openai/gpt-5.6-luna", "openai/gpt-6-sol", "openai/gpt-6-luna", + ), + _models( + _OPENROUTER_OPENAI_MAX_MANDATORY, "openai/gpt-6-astra", "openai/gpt-6.1-sol", ), ), "gemini": _surface( _models(_GEMINI_25_PRO, "gemini-2.5-pro"), - _models(_GEMINI_25_FLASH, "gemini-2.5-flash", "gemini-2.5-flash-lite"), + _models(_GEMINI_25_FLASH, "gemini-2.5-flash"), + _models(_GEMINI_25_FLASH_LITE, "gemini-2.5-flash-lite"), _models( - _GEMINI_3_HIGH_DEFAULT, + replace(_GEMINI_3_WITH_MINIMAL, provider_default="high"), "gemini-3-flash-preview", - "gemini-3.1-pro-preview", - "gemini-3.1-pro-preview-customtools", ), _models( - _GEMINI_3_MEDIUM_DEFAULT, - "gemini-3.5-flash", - "gemini-3.6-flash", - "gemini-3.7-flash", - "gemini-3.8-flash", + replace(_GEMINI_3_WITHOUT_MINIMAL, provider_default="high"), + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", ), + _models(_GEMINI_3_WITH_MINIMAL, "gemini-3.5-flash", "gemini-3.6-flash"), + _models(_GEMINI_3_WITHOUT_MINIMAL, "gemini-3.7-flash", "gemini-3.8-flash"), _models( - _GEMINI_3_MINIMAL_DEFAULT, "gemini-3.1-flash-lite", "gemini-3.5-flash-lite" + replace(_GEMINI_3_WITH_MINIMAL, provider_default="minimal"), + "gemini-3.1-flash-lite", + "gemini-3.5-flash-lite", ), ), "grok": _surface( @@ -735,13 +1008,11 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: ), "anthropic": _surface( _models(_CLAUDE_46, "claude-sonnet-4-6", "claude-opus-4-6"), + _models(_CLAUDE_47_48, "claude-opus-4-7", "claude-opus-4-8"), + _models(_CLAUDE_5, "claude-sonnet-5", "claude-opus-5"), _models( - _CLAUDE_47_PLUS, - "claude-opus-4-7", - "claude-opus-4-8", - "claude-sonnet-5", + _CLAUDE_ALWAYS_THINKING, "claude-sonnet-5-5", - "claude-opus-5", "claude-fable-5", "claude-fable-5-1", "claude-mythos-5", @@ -790,17 +1061,23 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: *_bedrock_ids("anthropic.claude-opus-4-6-v1", "us", "eu", "au", "global"), ), _models( - _BEDROCK_CLAUDE_47_PLUS, + _BEDROCK_CLAUDE_47_48, *_bedrock_ids( "anthropic.claude-opus-4-7", "us", "eu", "jp", "au", "global" ), *_bedrock_ids( "anthropic.claude-opus-4-8", "us", "eu", "jp", "au", "global" ), + ), + _models( + _BEDROCK_CLAUDE_5, *_bedrock_ids( "anthropic.claude-sonnet-5", "us", "eu", "au", "in", "global" ), *_bedrock_ids("anthropic.claude-opus-5", "us", "eu", "au", "in", "global"), + ), + _models( + _BEDROCK_CLAUDE_ALWAYS_THINKING, *_bedrock_ids("anthropic.claude-fable-5", "us", "global"), *_bedrock_ids("anthropic.claude-fable-5-1", "us", "global"), ), diff --git a/agent_core/core/session/session.py b/agent_core/core/session/session.py index 0ef00b6f..3fde370c 100644 --- a/agent_core/core/session/session.py +++ b/agent_core/core/session/session.py @@ -16,6 +16,7 @@ from datetime import datetime from typing import List, Dict, Any, Optional +from agent_core.core.models.reasoning import ReasoningChoice from agent_core.core.session.todo import TodoItem @@ -34,6 +35,15 @@ class SessionType: MAIN_SESSION_ID = "main" +def _stored_reasoning_effort(value: Any) -> Optional[str]: + """A persisted reasoning choice, or None when absent or no longer valid. + + A None is reseeded with the model's default level by the session manager + when the session is restored. + """ + return value if value in {choice.value for choice in ReasoningChoice} else None + + @dataclass class Session: """ @@ -54,6 +64,10 @@ class Session: workspace_dir: Persistent scratch directory for this session. agent_app_project_id: Backing project id for agent_app sessions. gui_mode: Whether this session drives the GUI action space. + reasoning_effort: The session's reasoning choice (a ReasoningChoice + value, picked in the chat input). The session manager seeds it + with the model's default level when the session is created or + restored without a valid one; None only until then. action_count/token_count: Budget counters for the current run (reset when a new run starts). input_tokens/output_tokens/cache_tokens: LLM usage breakdown for @@ -79,6 +93,7 @@ class Session: workspace_dir: Optional[str] = None agent_app_project_id: Optional[str] = None gui_mode: bool = False + reasoning_effort: Optional[str] = None # Per-run budget counters action_count: int = 0 token_count: int = 0 @@ -140,6 +155,7 @@ def to_dict(self) -> Dict[str, Any]: "workspace_dir": self.workspace_dir, "agent_app_project_id": self.agent_app_project_id, "gui_mode": self.gui_mode, + "reasoning_effort": self.reasoning_effort, "action_count": self.action_count, "token_count": self.token_count, "input_tokens": self.input_tokens, @@ -168,6 +184,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Session": workspace_dir=data.get("workspace_dir"), agent_app_project_id=data.get("agent_app_project_id"), gui_mode=data.get("gui_mode", False), + reasoning_effort=_stored_reasoning_effort(data.get("reasoning_effort")), action_count=data.get("action_count", 0), token_count=data.get("token_count", 0), input_tokens=data.get("input_tokens", 0), diff --git a/agent_core/core/state/session.py b/agent_core/core/state/session.py index 3c51974a..156d19e7 100644 --- a/agent_core/core/state/session.py +++ b/agent_core/core/state/session.py @@ -20,12 +20,20 @@ # At session deletion: StateSession.end(session_id) + + # Around work done for a session (its serial loop, or a call made for it + # outside the loop), so context-scoped consumers can find it: + with StateSession.bind(session_id): + ... + state = StateSession.bound() # None outside any bound session """ from __future__ import annotations +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass, field -from typing import ClassVar, Optional, Dict, Any, TYPE_CHECKING +from typing import ClassVar, Iterator, Optional, Dict, Any, TYPE_CHECKING from agent_core.core.state.types import AgentProperties @@ -33,6 +41,14 @@ from agent_core.core.session.session import Session +#: Id of the session whose work runs in the current context. asyncio tasks +#: and asyncio.to_thread inherit it, so everything a session's loop starts +#: (actions, sub-agents, LLM calls) sees the session that started it. +_bound_session_id: ContextVar[Optional[str]] = ContextVar( + "bound_session_id", default=None +) + + @dataclass class StateSession: """Per-session runtime state isolated from other concurrent sessions. @@ -128,6 +144,21 @@ def end(cls, session_id: str) -> None: """Remove a session's state (session deletion).""" cls._instances.pop(session_id, None) + @classmethod + @contextmanager + def bind(cls, session_id: str) -> Iterator[None]: + """Mark the current context as doing work for ``session_id``.""" + token = _bound_session_id.set(session_id) + try: + yield + finally: + _bound_session_id.reset(token) + + @classmethod + def bound(cls) -> Optional["StateSession"]: + """State of the session the current context works for, if any.""" + return cls.get_or_none(_bound_session_id.get()) + @classmethod def get_all_session_ids(cls) -> list[str]: """Get all active session IDs.""" diff --git a/app/agent_base.py b/app/agent_base.py index e6157d20..9b2cc7f2 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -28,6 +28,7 @@ import traceback import time import json +from contextlib import nullcontext from dataclasses import dataclass from typing import Awaitable, Callable, Dict, Iterable, Optional @@ -86,6 +87,7 @@ MemoryFileWatcher, LLMCallType, ) +from agent_core.core.models.reasoning import ReasoningChoice from agent_core.core.session import Session, SessionType, MAIN_SESSION_ID from agent_core.core.state.session import StateSession from app.context_engine import ContextEngine @@ -502,10 +504,21 @@ def get_commands(self) -> Dict[str, AgentCommand]: # Session API (sidebar surface) # ===================================== - def create_chat_session(self, title: str = "New chat") -> Session: - """Create a fresh chat session (the "+ New Chat" button).""" + def create_chat_session( + self, + title: str = "New chat", + reasoning_effort: Optional[ReasoningChoice] = None, + ) -> Session: + """Create a fresh chat session (the "+ New Chat" button). + + ``reasoning_effort`` carries the draft chat's picker value into the + session the draft becomes; None (picker untouched) starts it at the + default level of the model in use. + """ return self.session_manager.create_session( - session_type=SessionType.CHAT, title=title + session_type=SessionType.CHAT, + title=title, + reasoning_effort=reasoning_effort, ) async def delete_session(self, session_id: str) -> bool: @@ -528,6 +541,12 @@ def rename_session(self, session_id: str, title: str) -> bool: """Rename a session's sidebar title.""" return self.session_manager.rename_session(session_id, title) + def set_session_reasoning_effort( + self, session_id: str, choice: ReasoningChoice + ) -> bool: + """Set a session's reasoning choice (the chat input's picker).""" + return self.session_manager.set_reasoning_effort(session_id, choice) + # ===================================== # Main Agent Cycle # ===================================== @@ -1625,15 +1644,18 @@ async def _auto_title_session( title = "" try: - response = await self.llm.generate_response_async( - system_prompt=( - "Generate a concise 2-5 word title for a conversation " - "that starts with the user request below. Reply with a " - 'JSON object: {"title": ""}. Same language ' - "as the request, no punctuation at the end." - ), - user_prompt=basis[:2000], - ) + # Titling is work for this session: it reasons as the session's + # picker says, also when started outside the session's loop. + with StateSession.bind(session_id): + response = await self.llm.generate_response_async( + system_prompt=( + "Generate a concise 2-5 word title for a conversation " + "that starts with the user request below. Reply with a " + 'JSON object: {"title": ""}. Same language ' + "as the request, no punctuation at the end." + ), + user_prompt=basis[:2000], + ) title = self._parse_session_title(response) except Exception as e: logger.debug(f"[SESSION] Auto-title LLM call failed for {session_id}: {e}") @@ -2597,15 +2619,31 @@ async def _handle_external_event(self, payload: Dict) -> None: except Exception as e: logger.error(f"Error handling external event: {e}", exc_info=True) - async def _handle_prompt_enhance(self, user_message: str) -> str: + async def _handle_prompt_enhance( + self, + user_message: str, + session_id: Optional[str], + reasoning_choice: Optional[ReasoningChoice], + ) -> str: + """Rewrite a prompt typed in the chat input for clarity. + + The call reasons like the chat the prompt is typed in: the session + ``session_id``, or, for a draft chat that has no session yet + (``session_id`` None), its picker value ``reasoning_choice``. + """ try: from agent_core.core.prompts.reasoning import ( PROMPT_ENHANCE_REASONING_PROMPT, ) - response = await self.llm.generate_response_async( - system_prompt=PROMPT_ENHANCE_REASONING_PROMPT, user_prompt=user_message - ) + with ( + StateSession.bind(session_id) if session_id is not None else nullcontext() + ): + response = await self.llm.generate_response_async( + system_prompt=PROMPT_ENHANCE_REASONING_PROMPT, + user_prompt=user_message, + reasoning_choice=reasoning_choice, + ) result = json.loads(response) return result.get("enhanced_prompt", "") except Exception as e: diff --git a/app/triggers/runtime.py b/app/triggers/runtime.py index 48e44478..68e8febb 100644 --- a/app/triggers/runtime.py +++ b/app/triggers/runtime.py @@ -26,6 +26,7 @@ QueueClosed, ) from agent_core.core.session import MAIN_SESSION_ID +from agent_core.core.state.session import StateSession from app.triggers.sources import TriggerSource if TYPE_CHECKING: @@ -409,7 +410,10 @@ async def _consume(self, session_id: str, queue: SessionTriggerQueue) -> None: ) logger.info(f"[SessionRuntime] Loop started for session {session_id}") - with session_ctx: + # Bind the session for everything its turns run (actions, sub-agents, + # LLM calls): the LLM layer reads the session's reasoning choice + # through this binding. + with session_ctx, StateSession.bind(session_id): while self._running: try: trig = await queue.get() diff --git a/app/ui_layer/adapters/browser_adapter.py b/app/ui_layer/adapters/browser_adapter.py index 2a171d2a..ba33a1ce 100644 --- a/app/ui_layer/adapters/browser_adapter.py +++ b/app/ui_layer/adapters/browser_adapter.py @@ -24,6 +24,7 @@ SCHEDULE_HOUR_DEFAULT, SCHEDULE_MINUTE_DEFAULT, ) +from agent_core.core.models.reasoning import ReasoningChoice from agent_core.utils.logger import logger from app.config import AGENT_WORKSPACE_ROOT, APP_DATA_PATH from app.i18n import tui @@ -1005,11 +1006,20 @@ async def submit_message( client_id=client_id, ) - async def _handle_enhance_prompt(self, content: str, ws) -> None: - """Enhance a user's prompt using the LLM for clarity and precision.""" + async def _handle_enhance_prompt(self, data: Dict[str, Any], ws) -> None: + """Enhance a user's prompt using the LLM for clarity and precision. + + The call reasons like the chat the prompt is typed in: its session, + or a draft chat's picker value. + """ + session_id = data.get("sessionId") or "main" try: enhanced: str = await self._controller.handle_prompt_enhance( - user_message=content + user_message=data["content"], + session_id=None if session_id == "new" else session_id, + reasoning_choice=( + self._draft_reasoning(data) if session_id == "new" else None + ), ) await self._send_to( ws, {"type": "prompt_enhanced", "content": enhanced.strip()} @@ -1451,7 +1461,9 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: # session_created is broadcast (with the sender's clientId) before # the message so the draft view can navigate to the real session. if session_id == "new": - session = self._controller.agent.create_chat_session() + session = self._controller.agent.create_chat_session( + reasoning_effort=self._draft_reasoning(data) + ) session_id = session.id await self._broadcast( { @@ -1509,7 +1521,9 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: name = command.strip().split()[0].lower() if command.strip() else "" cmd = self._controller.command_registry.get(name) if name else None if cmd is not None and cmd.requires_session: - session = self._controller.agent.create_chat_session() + session = self._controller.agent.create_chat_session( + reasoning_effort=self._draft_reasoning(data) + ) session_id = session.id await self._broadcast( { @@ -1530,7 +1544,7 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: elif msg_type == "enhance_prompt": content = data.get("content", "") if content and ws: - await self._handle_enhance_prompt(content, ws) + await self._handle_enhance_prompt(data, ws) elif msg_type == "chat_history": session_id = data.get("sessionId") or "main" @@ -1551,6 +1565,12 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: elif msg_type == "session_list": await self._handle_session_list(ws) + elif msg_type == "session_reasoning_set": + await self._handle_session_reasoning_set(data) + + elif msg_type == "reasoning_options_get": + await self._handle_reasoning_options_get(ws) + # File operations elif msg_type == "file_list": directory = data.get("directory", "") @@ -4270,8 +4290,16 @@ def _session_info(session) -> Dict[str, Any]: "createdAt": session.created_at, "lastActiveAt": session.last_active_at, "agentAppProjectId": session.agent_app_project_id, + "reasoningEffort": session.reasoning_effort, } + @staticmethod + def _draft_reasoning(data: Dict[str, Any]) -> Optional[ReasoningChoice]: + """A draft chat's picker value from a message; None when untouched + (the model's default level then applies).""" + value = data.get("reasoningEffort") + return ReasoningChoice(value) if value is not None else None + async def _handle_session_delete(self, data: Dict[str, Any]) -> None: """Delete a session and its chat history. The main session is permanent.""" from agent_core.core.session import MAIN_SESSION_ID @@ -4310,6 +4338,49 @@ async def _handle_session_rename(self, data: Dict[str, Any]) -> None: except Exception as e: logger.error(f"[SESSION] Rename failed for {session_id}: {e}") + async def _handle_session_reasoning_set(self, data: Dict[str, Any]) -> None: + """Set a session's reasoning choice (the chat input's picker).""" + session_id = (data.get("sessionId") or "").strip() + if not session_id: + return + try: + choice = ReasoningChoice(data.get("reasoningEffort")) + except ValueError: + logger.warning( + f"[SESSION] Unknown reasoning choice {data.get('reasoningEffort')!r} " + f"for {session_id}" + ) + return + if self._controller.agent.set_session_reasoning_effort(session_id, choice): + await self.broadcast_session_updated(session_id) + + async def _handle_reasoning_options_get(self, ws) -> None: + """Send what the reasoning picker offers for the model in use. + + Read from the live LLM interface, so it reflects the model actually + called (after any subscription substitution), not only the settings. + """ + llm = self._controller.agent.llm + options = llm.reasoning_options() + data: Dict[str, Any] = { + "success": True, + "model": llm.model, + "configurable": options is not None, + } + if options is not None: + data.update( + { + "choices": [choice.value for choice in options.choices], + "defaultLevel": options.default_level, + "providerDefault": options.provider_default, + "resolution": { + requested.value: effective.value + for requested, effective in options.resolution.items() + }, + } + ) + await self._send_to(ws, {"type": "reasoning_options_get", "data": data}) + async def _handle_session_clear(self, data: Dict[str, Any]) -> None: """Clear a session's conversation (chat + activity rows and agent-side state).""" diff --git a/app/ui_layer/browser/frontend/src/components/Chat/Chat.module.css b/app/ui_layer/browser/frontend/src/components/Chat/Chat.module.css index 30b2c6ad..8bd917dc 100644 --- a/app/ui_layer/browser/frontend/src/components/Chat/Chat.module.css +++ b/app/ui_layer/browser/frontend/src/components/Chat/Chat.module.css @@ -206,6 +206,12 @@ gap: var(--space-2); } +.controlsLeft { + display: flex; + align-items: center; + gap: var(--space-2); +} + .controlsRight { display: flex; align-items: center; diff --git a/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx b/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx index f56cf906..17da1cb6 100644 --- a/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx +++ b/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx @@ -41,6 +41,7 @@ import { selectPendingQuestions, } from '../../store/selectors/messages' import { QuestionBox } from './QuestionBox' +import { ReasoningPicker } from './ReasoningPicker' import { mergeTimeline, type TimelineEntry } from './timeline' import { selectSessionActivity } from '../../store/selectors/activity' import { selectSessionBusy, selectSessionRunState } from '../../store/selectors/agent' @@ -881,13 +882,13 @@ export function Chat({ sessionId, placeholder }: ChatProps) { const handleEnhancePrompt = useCallback(() => { if (!input.trim() || enhancing) return setEnhancing(true) - enhancePrompt(input.trim()) + enhancePrompt(input.trim(), sessionId) enhanceTimeoutRef.current = setTimeout(() => { enhanceTimeoutRef.current = null setEnhancing(false) showToast('error', t('chat:toast.enhanceTimedOut')) }, ENHANCE_TIMEOUT_MS) - }, [input, enhancing, enhancePrompt, showToast, t]) + }, [input, enhancing, enhancePrompt, sessionId, showToast, t]) useEffect(() => { return () => { @@ -1519,41 +1520,44 @@ export function Chat({ sessionId, placeholder }: ChatProps) { />
-
- - {plusOpen && ( -
- - -
- )} +
+
+ + {plusOpen && ( +
+ + +
+ )} +
+
diff --git a/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.module.css b/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.module.css new file mode 100644 index 00000000..e0fcd880 --- /dev/null +++ b/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.module.css @@ -0,0 +1,102 @@ +/* Reasoning picker beside the "+" button; mirrors the "+" menu styling. */ +.wrap { + position: relative; +} + +.button { + display: flex; + align-items: center; + gap: var(--space-1); + height: 30px; + padding: 0 10px; + border-radius: 15px; + background: transparent; + border: 1px solid var(--border-primary); + color: var(--text-secondary); + font-family: inherit; + font-size: var(--text-xs); + white-space: nowrap; + cursor: pointer; + transition: background var(--transition-fast), color var(--transition-fast); +} + +.button:hover:not(:disabled) { + background: var(--bg-tertiary); + color: var(--text-primary); +} + +.button:disabled { + opacity: 0.45; + cursor: not-allowed; +} + +.menu { + position: absolute; + bottom: calc(100% + 8px); + left: 0; + min-width: 220px; + padding: 4px; + background: var(--bg-secondary); + border: 1px solid var(--border-primary); + border-radius: var(--radius-md); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); + z-index: 999; +} + +.header { + padding: 6px 10px 4px; + color: var(--text-tertiary); + font-size: 11px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.item { + display: flex; + align-items: center; + gap: var(--space-2); + width: 100%; + padding: 7px 10px; + background: transparent; + border: none; + border-radius: var(--radius-sm); + color: var(--text-secondary); + font-family: inherit; + font-size: var(--text-sm); + cursor: pointer; + text-align: left; +} + +.item:hover { + background: var(--bg-tertiary); + color: var(--text-primary); +} + +.itemActive { + color: var(--text-primary); +} + +.check { + display: flex; + width: 14px; + flex-shrink: 0; +} + +.itemLabel { + flex: 1; +} + +.itemDetail { + color: var(--text-tertiary); + font-size: 11px; +} + +.note { + margin-top: 4px; + padding: 6px 10px 4px; + border-top: 1px solid var(--border-primary); + color: var(--text-tertiary); + font-size: 11px; + line-height: 1.4; +} diff --git a/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.tsx b/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.tsx new file mode 100644 index 00000000..3e50a23a --- /dev/null +++ b/app/ui_layer/browser/frontend/src/components/Chat/ReasoningPicker.tsx @@ -0,0 +1,159 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { Brain, Check, ChevronDown } from 'lucide-react' +import type { ReasoningChoice } from '../../types' +import { useWebSocket } from '../../contexts/WebSocketContext' +import { usePersistedState } from '../../hooks' +import { useAppSelector } from '../../store/hooks' +import { RESOURCES, useResource } from '../../store/resources' +import { selectReasoningOptions } from '../../store/selectors/reasoning' +import { selectSessionById } from '../../store/selectors/sessions' +import { UI_STATE } from '../../store/uiState' +import styles from './ReasoningPicker.module.css' + +const CHOICE_LABEL_KEYS = { + provider_default: 'chat:reasoning.choice.provider_default', + off: 'chat:reasoning.choice.off', + minimal: 'chat:reasoning.choice.minimal', + low: 'chat:reasoning.choice.low', + medium: 'chat:reasoning.choice.medium', + high: 'chat:reasoning.choice.high', + xhigh: 'chat:reasoning.choice.xhigh', + max: 'chat:reasoning.choice.max', +} as const satisfies Record + +const isReasoningChoice = (value: string): value is ReasoningChoice => + Object.prototype.hasOwnProperty.call(CHOICE_LABEL_KEYS, value) + +interface ReasoningPickerProps { + /** The chat this composer belongs to ('new' for the draft view). */ + sessionId: string +} + +/** + * How hard the model reasons for this chat (chat input, beside "+"). + * + * Follows the pi agent harness: each chat holds a concrete choice, and the + * menu lists only what the model in use accepts, with the model's default + * level marked "Default" (where every new chat starts). A real session's + * choice is saved server-side; the draft view keeps it in UI state until its + * first message creates the session (untouched: the default level). A stored + * choice the model lacks shows the level actually used instead (the backend + * clamps it the way pi does and keeps the stored choice for other models). + */ +export function ReasoningPicker({ sessionId }: ReasoningPickerProps) { + const { t } = useTranslation(['chat', 'common']) + const { setSessionReasoning } = useWebSocket() + useResource(RESOURCES.reasoningOptions) + const options = useAppSelector(selectReasoningOptions) + const session = useAppSelector(state => selectSessionById(state, sessionId)) + const [draftChoice] = usePersistedState(UI_STATE.chat.draftReasoningEffort) + const [open, setOpen] = useState(false) + const wrapRef = useRef(null) + + // The chat's own choice; null while it has none yet (an untouched draft, + // or the main session before the backend has sent it). + const stored: ReasoningChoice | null = + sessionId === 'new' ? draftChoice : (session?.reasoningEffort ?? null) + + useEffect(() => { + if (!open) return + const handler = (e: MouseEvent) => { + if (wrapRef.current && !wrapRef.current.contains(e.target as Node)) setOpen(false) + } + document.addEventListener('mousedown', handler) + return () => document.removeEventListener('mousedown', handler) + }, [open]) + + const choiceLabel = (value: ReasoningChoice): string => t(CHOICE_LABEL_KEYS[value]) + + // What a level-like value from the backend reads as: a choice, or the + // provider deciding the amount itself. + const levelLabel = (value: string): string => + isReasoningChoice(value) ? choiceLabel(value) : t('chat:reasoning.dynamic') + + if (!options || !options.configurable) { + return ( +
+ +
+ ) + } + + const { defaultLevel, providerDefault } = options + const requested: ReasoningChoice = stored ?? defaultLevel + const effective = options.resolution[requested] + // A choice as it takes effect on this model. + const summary = (value: ReasoningChoice): string => + value === 'provider_default' + ? t('chat:reasoning.withLevel', { + choice: choiceLabel('provider_default'), + level: levelLabel(providerDefault), + }) + : choiceLabel(value) + // The row marked as current: the requested choice, or what it resolves to + // when the model does not offer it. + const marked = options.choices.includes(requested) ? requested : effective + // Only a level the model lacks is clamped; provider_default on a model + // whose default is no reasoning simply IS off. + const clamped = !options.choices.includes(requested) && requested !== 'provider_default' + + return ( +
+ + {open && ( +
+
{t('chat:reasoning.header', { model: options.model })}
+ {options.choices.map(value => ( + + ))} + {clamped && ( +
+ {t('chat:reasoning.clamped', { requested: choiceLabel(requested), effective: summary(effective) })} +
+ )} +
+ )} +
+ ) +} diff --git a/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx b/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx index 773d09f7..ba88eabd 100644 --- a/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx +++ b/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx @@ -680,6 +680,7 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { title: 'Main', createdAt: '', lastActiveAt: '', + reasoningEffort: null, } return ( diff --git a/app/ui_layer/browser/frontend/src/contexts/WebSocketContext.tsx b/app/ui_layer/browser/frontend/src/contexts/WebSocketContext.tsx index 6248a6cc..ffaba7bb 100644 --- a/app/ui_layer/browser/frontend/src/contexts/WebSocketContext.tsx +++ b/app/ui_layer/browser/frontend/src/contexts/WebSocketContext.tsx @@ -3,7 +3,7 @@ import { useNavigate } from 'react-router-dom' import { useStore } from 'react-redux' import type { ChatMessage, SessionInfo, WSMessage, MetricsTimePeriod, - AgentAppCreateRequest, + AgentAppCreateRequest, ReasoningChoice, } from '../types' import { QUESTION_DISMISSED } from '../types' import i18n from '../i18n/config' @@ -36,6 +36,8 @@ import { markStopping as agentAppMarkStopping, } from '../store/slices/agentAppSlice' import { setStatus, setSessionRunState } from '../store/slices/agentSlice' +import { upsertSession } from '../store/slices/sessionsSlice' +import { selectSessionById } from '../store/selectors/sessions' import { setUiState } from '../store/slices/uiSlice' import { selectUiState } from '../store/selectors/ui' import { UI_STATE } from '../store/uiState' @@ -114,6 +116,9 @@ interface WebSocketContextType extends WebSocketState { deleteSession: (sessionId: string) => void renameSession: (sessionId: string, title: string) => void clearSession: (sessionId: string) => void + // Reasoning picker: a session's choice (the draft view keeps it locally + // until its first message creates the session) + setSessionReasoning: (sessionId: string, choice: ReasoningChoice) => void requestChatHistory: (sessionId: string, beforeTimestamp?: number, limit?: number) => void // Per-session unread tracking (read with UI_STATE.chat.lastSeenMessageIds) markSessionSeen: (sessionId: string) => void @@ -127,8 +132,8 @@ interface WebSocketContextType extends WebSocketState { submitOnboardingStep: (value: string | string[] | Record) => void skipOnboardingStep: () => void goBackOnboardingStep: () => void - // Enhance prompt - enhancePrompt: (content: string) => void + // Enhance prompt (reasons like the chat it is typed in) + enhancePrompt: (content: string, sessionId: string) => void clearEnhancedPrompt: () => void // Local LLM (Ollama) methods checkLocalLLM: () => void @@ -240,6 +245,9 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { sessionId: session.id, state: startsRun === false ? 'idle' : 'running', })) + // The session now holds the draft's reasoning choice; the next + // draft starts untouched (the model's default level) again. + dispatch(setUiState(UI_STATE.chat.draftReasoningEffort, null)) navigateRef.current(`/session/${session.id}`, { replace: true }) } break @@ -281,6 +289,19 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { for (const envelope of expired) rollbackExpiredSend(envelope, dispatch) }), [dispatch, showToast]) + // A draft chat's reasoning choice rides along with whatever turns the + // draft into a session (first message, session command) or calls the LLM + // for it (prompt enhance); a real session's choice is read server-side. + // An untouched draft sends none: the model's default level applies. + const draftReasoning = useCallback( + (sessionId: string): { reasoningEffort?: ReasoningChoice } => { + if (sessionId !== 'new') return {} + const choice = selectUiState(store.getState(), UI_STATE.chat.draftReasoningEffort) + return choice === null ? {} : { reasoningEffort: choice } + }, + [store], + ) + const sendMessage = useCallback(( content: string, attachments: PendingAttachment[] | undefined, @@ -332,8 +353,9 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { ), replyContext: replyContext || null, clientId, + ...draftReasoning(sessionId), })) - }, [sendOrQueue, dispatch]) + }, [sendOrQueue, dispatch, draftReasoning]) const sendCommand = useCallback((command: string, sessionId: string) => { const clientId = newClientId() @@ -348,8 +370,10 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { pendingDraftClientIdsRef.current.add(clientId) } - sendOrQueue(JSON.stringify({ type: 'command', command, sessionId, clientId })) - }, [sendOrQueue]) + sendOrQueue(JSON.stringify({ + type: 'command', command, sessionId, clientId, ...draftReasoning(sessionId), + })) + }, [sendOrQueue, draftReasoning]) // Force-stop a session's in-flight run (chat input's stop button). // Optimistically enters 'stopping' so the button spins instantly; the @@ -374,6 +398,18 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { sendOrQueue(JSON.stringify({ type: 'session_clear', sessionId })) }, [sendOrQueue]) + const setSessionReasoning = useCallback((sessionId: string, choice: ReasoningChoice) => { + if (sessionId === 'new') { + dispatch(setUiState(UI_STATE.chat.draftReasoningEffort, choice)) + return + } + // Optimistic: the picker reflects the choice at once; the server's + // session_updated broadcast is authoritative. + const session = selectSessionById(store.getState(), sessionId) + if (session) dispatch(upsertSession({ ...session, reasoningEffort: choice })) + sendOrQueue(JSON.stringify({ type: 'session_reasoning_set', sessionId, reasoningEffort: choice })) + }, [sendOrQueue, dispatch, store]) + const requestChatHistory = useCallback(( sessionId: string, beforeTimestamp?: number, @@ -407,9 +443,11 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { dispatch(setUiState(UI_STATE.chat.lastSeenMessageIds, { ...seen, [sessionId]: lastId })) }, [dispatch, store]) - const enhancePrompt = useCallback((content: string) => { - sendOrQueue(JSON.stringify({ type: 'enhance_prompt', content })) - }, [sendOrQueue]) + const enhancePrompt = useCallback((content: string, sessionId: string) => { + sendOrQueue(JSON.stringify({ + type: 'enhance_prompt', content, sessionId, ...draftReasoning(sessionId), + })) + }, [sendOrQueue, draftReasoning]) const clearEnhancedPrompt = useCallback(() => { setState(prev => ({ ...prev, enhancedPrompt: null })) @@ -616,6 +654,7 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { deleteSession, renameSession, clearSession, + setSessionReasoning, requestChatHistory, markSessionSeen, openFile, @@ -648,7 +687,7 @@ export function WebSocketProvider({ children }: { children: ReactNode }) { updateAgentAppTheme, }), [ state, sendMessage, sendCommand, stopSession, deleteSession, renameSession, clearSession, - requestChatHistory, markSessionSeen, openFile, openFolder, requestFilteredMetrics, + setSessionReasoning, requestChatHistory, markSessionSeen, openFile, openFolder, requestFilteredMetrics, subscribeDashboardMetrics, unsubscribeDashboardMetrics, requestOnboardingStep, submitOnboardingStep, skipOnboardingStep, goBackOnboardingStep, checkLocalLLM, testLocalLLMConnection, installLocalLLM, startLocalLLM, requestSuggestedModels, pullOllamaModel, diff --git a/app/ui_layer/browser/frontend/src/locales/en/chat.json b/app/ui_layer/browser/frontend/src/locales/en/chat.json index 58d1b790..e6b429ea 100644 --- a/app/ui_layer/browser/frontend/src/locales/en/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/en/chat.json @@ -31,6 +31,26 @@ "stopRun": "Stop run", "stoppingRun": "Stopping run" }, + "reasoning": { + "title": "Reasoning", + "picker": "Reasoning effort", + "header": "Reasoning · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "Model decides", + "defaultTag": "Default", + "notConfigurable": "Reasoning isn't adjustable for {{model}}", + "clamped": "{{requested}} isn't available on this model, so {{effective}} is used.", + "choice": { + "provider_default": "Provider default", + "off": "Off", + "minimal": "Minimal", + "low": "Low", + "medium": "Medium", + "high": "High", + "xhigh": "Extra high", + "max": "Max" + } + }, "reply": { "replyingTo": "Replying to: <1>{{name}}", "cancel": "Cancel reply" diff --git a/app/ui_layer/browser/frontend/src/locales/es/chat.json b/app/ui_layer/browser/frontend/src/locales/es/chat.json index 2236e221..9713e856 100644 --- a/app/ui_layer/browser/frontend/src/locales/es/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/es/chat.json @@ -31,6 +31,26 @@ "stopRun": "Detener ejecución", "stoppingRun": "Deteniendo la ejecución" }, + "reasoning": { + "title": "Razonamiento", + "picker": "Nivel de razonamiento", + "header": "Razonamiento · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "Lo decide el modelo", + "defaultTag": "Predeterminado", + "notConfigurable": "El razonamiento no es ajustable en {{model}}", + "clamped": "{{requested}} no está disponible en este modelo, así que se usa {{effective}}.", + "choice": { + "provider_default": "Predeterminado del proveedor", + "off": "Desactivado", + "minimal": "Mínimo", + "low": "Bajo", + "medium": "Medio", + "high": "Alto", + "xhigh": "Muy alto", + "max": "Máximo" + } + }, "reply": { "replyingTo": "Respondiendo a: <1>{{name}}", "cancel": "Cancelar respuesta" diff --git a/app/ui_layer/browser/frontend/src/locales/id/chat.json b/app/ui_layer/browser/frontend/src/locales/id/chat.json index 2c2ec0c2..dfe7e89e 100644 --- a/app/ui_layer/browser/frontend/src/locales/id/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/id/chat.json @@ -31,6 +31,26 @@ "stopRun": "Hentikan proses", "stoppingRun": "Menghentikan proses" }, + "reasoning": { + "title": "Penalaran", + "picker": "Tingkat penalaran", + "header": "Penalaran · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "Ditentukan model", + "defaultTag": "Bawaan", + "notConfigurable": "Penalaran tidak dapat diatur untuk {{model}}", + "clamped": "{{requested}} tidak tersedia pada model ini, jadi {{effective}} yang digunakan.", + "choice": { + "provider_default": "Bawaan penyedia", + "off": "Mati", + "minimal": "Minimal", + "low": "Rendah", + "medium": "Sedang", + "high": "Tinggi", + "xhigh": "Sangat tinggi", + "max": "Maksimum" + } + }, "reply": { "replyingTo": "Membalas: <1>{{name}}", "cancel": "Batalkan balasan" diff --git a/app/ui_layer/browser/frontend/src/locales/ja/chat.json b/app/ui_layer/browser/frontend/src/locales/ja/chat.json index b6125efe..743c3719 100644 --- a/app/ui_layer/browser/frontend/src/locales/ja/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/ja/chat.json @@ -31,6 +31,26 @@ "stopRun": "実行を停止", "stoppingRun": "実行を停止中" }, + "reasoning": { + "title": "推論", + "picker": "推論の強度", + "header": "推論 · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "モデルが決定", + "defaultTag": "既定", + "notConfigurable": "{{model}} では推論を調整できません", + "clamped": "{{requested}} はこのモデルでは使えないため、{{effective}} を使用します。", + "choice": { + "provider_default": "プロバイダーの既定", + "off": "オフ", + "minimal": "最小", + "low": "低", + "medium": "中", + "high": "高", + "xhigh": "超高", + "max": "最大" + } + }, "reply": { "replyingTo": "返信先: <1>{{name}}", "cancel": "返信をキャンセル" diff --git a/app/ui_layer/browser/frontend/src/locales/ko/chat.json b/app/ui_layer/browser/frontend/src/locales/ko/chat.json index d2f242ac..7fcd2224 100644 --- a/app/ui_layer/browser/frontend/src/locales/ko/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/ko/chat.json @@ -31,6 +31,26 @@ "stopRun": "실행 중지", "stoppingRun": "실행 중지하는 중" }, + "reasoning": { + "title": "추론", + "picker": "추론 강도", + "header": "추론 · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "모델이 결정", + "defaultTag": "기본값", + "notConfigurable": "{{model}}에서는 추론을 조정할 수 없습니다", + "clamped": "이 모델에서는 {{requested}}을(를) 사용할 수 없어 {{effective}}을(를) 사용합니다.", + "choice": { + "provider_default": "제공자 기본값", + "off": "끄기", + "minimal": "최소", + "low": "낮음", + "medium": "보통", + "high": "높음", + "xhigh": "매우 높음", + "max": "최대" + } + }, "reply": { "replyingTo": "답장 대상: <1>{{name}}", "cancel": "답장 취소" diff --git a/app/ui_layer/browser/frontend/src/locales/zh-CN/chat.json b/app/ui_layer/browser/frontend/src/locales/zh-CN/chat.json index df7a5b04..286c7ea4 100644 --- a/app/ui_layer/browser/frontend/src/locales/zh-CN/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/zh-CN/chat.json @@ -31,6 +31,26 @@ "stopRun": "停止运行", "stoppingRun": "正在停止运行" }, + "reasoning": { + "title": "推理", + "picker": "推理强度", + "header": "推理 · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "由模型决定", + "defaultTag": "默认", + "notConfigurable": "{{model}} 不支持调整推理", + "clamped": "此模型不支持“{{requested}}”,因此使用“{{effective}}”。", + "choice": { + "provider_default": "服务商默认", + "off": "关闭", + "minimal": "最低", + "low": "低", + "medium": "中", + "high": "高", + "xhigh": "超高", + "max": "最高" + } + }, "reply": { "replyingTo": "正在回复:<1>{{name}}", "cancel": "取消回复" diff --git a/app/ui_layer/browser/frontend/src/locales/zh-TW/chat.json b/app/ui_layer/browser/frontend/src/locales/zh-TW/chat.json index 7a006d1c..fe46e78d 100644 --- a/app/ui_layer/browser/frontend/src/locales/zh-TW/chat.json +++ b/app/ui_layer/browser/frontend/src/locales/zh-TW/chat.json @@ -31,6 +31,26 @@ "stopRun": "停止執行", "stoppingRun": "正在停止執行" }, + "reasoning": { + "title": "推理", + "picker": "推理強度", + "header": "推理 · {{model}}", + "withLevel": "{{choice}} · {{level}}", + "dynamic": "由模型決定", + "defaultTag": "預設", + "notConfigurable": "{{model}} 不支援調整推理", + "clamped": "此模型不支援「{{requested}}」,因此使用「{{effective}}」。", + "choice": { + "provider_default": "供應商預設", + "off": "關閉", + "minimal": "最低", + "low": "低", + "medium": "中", + "high": "高", + "xhigh": "超高", + "max": "最高" + } + }, "reply": { "replyingTo": "正在回覆:<1>{{name}}", "cancel": "取消回覆" diff --git a/app/ui_layer/browser/frontend/src/store/index.ts b/app/ui_layer/browser/frontend/src/store/index.ts index 1da895c7..9927fe6d 100644 --- a/app/ui_layer/browser/frontend/src/store/index.ts +++ b/app/ui_layer/browser/frontend/src/store/index.ts @@ -17,6 +17,7 @@ import proactiveSettingsReducer from './slices/proactiveSettingsSlice' import agentAppSettingsReducer from './slices/agentAppSettingsSlice' import generalSettingsReducer from './slices/generalSettingsSlice' import modelSettingsReducer from './slices/modelSettingsSlice' +import reasoningReducer from './slices/reasoningSlice' import integrationsSettingsReducer from './slices/integrationsSettingsSlice' import chatInputReducer from './slices/chatInputSlice' import playbooksReducer from './slices/playbooksSlice' @@ -58,6 +59,7 @@ export const store = configureStore({ agentAppSettings: agentAppSettingsReducer, generalSettings: generalSettingsReducer, modelSettings: modelSettingsReducer, + reasoning: reasoningReducer, integrationsSettings: integrationsSettingsReducer, chatInput: chatInputReducer, playbooks: playbooksReducer, diff --git a/app/ui_layer/browser/frontend/src/store/resources/catalog.ts b/app/ui_layer/browser/frontend/src/store/resources/catalog.ts index b1333396..a822018f 100644 --- a/app/ui_layer/browser/frontend/src/store/resources/catalog.ts +++ b/app/ui_layer/browser/frontend/src/store/resources/catalog.ts @@ -151,6 +151,13 @@ export const RESOURCES = { resource: 'model_settings', request: (send) => send({ type: 'slow_mode_get' }), }, + /** Chat input reasoning picker: what the model in use offers. A model or + * sign-in change is a model_settings change, so it refetches with it. */ + reasoningOptions: { + key: 'reasoningOptions', + resource: 'model_settings', + request: (send) => send({ type: 'reasoning_options_get' }), + }, // ── Static catalogs: refreshed after a reconnect ── playbooks: { diff --git a/app/ui_layer/browser/frontend/src/store/selectors/reasoning.ts b/app/ui_layer/browser/frontend/src/store/selectors/reasoning.ts new file mode 100644 index 00000000..9bf4322b --- /dev/null +++ b/app/ui_layer/browser/frontend/src/store/selectors/reasoning.ts @@ -0,0 +1,3 @@ +import type { RootState } from '../index' + +export const selectReasoningOptions = (state: RootState) => state.reasoning.options diff --git a/app/ui_layer/browser/frontend/src/store/slices/reasoningSlice.ts b/app/ui_layer/browser/frontend/src/store/slices/reasoningSlice.ts new file mode 100644 index 00000000..fe229a39 --- /dev/null +++ b/app/ui_layer/browser/frontend/src/store/slices/reasoningSlice.ts @@ -0,0 +1,52 @@ +import { createSlice, PayloadAction } from '@reduxjs/toolkit' +import type { ReasoningChoice } from '../../types' +import { register } from '../socket/messageRegistry' + +// What the chat input's reasoning picker offers for the model in use. The +// backend computes it from the live LLM interface (reasoning_options_get), +// so it tracks model switches; the CHOICE is per session and lives on the +// session (SessionInfo.reasoningEffort) or, for a draft chat, in UI state. + +export type ReasoningOptions = + /** The model has no reasoning rule: nothing is adjustable. */ + | { configurable: false; model: string } + | { + configurable: true + model: string + /** Selectable choices in display order. */ + choices: ReasoningChoice[] + /** The model's default level (marked "Default"; new chats start here). */ + defaultLevel: ReasoningChoice + /** What 'provider_default' does: 'off', a level, or 'dynamic'. */ + providerDefault: string + /** Effective choice for every stored choice (pi-style clamping). */ + resolution: Record + } + +interface ReasoningState { + options: ReasoningOptions | null +} + +const initialState: ReasoningState = { + options: null, +} + +const reasoningSlice = createSlice({ + name: 'reasoning', + initialState, + reducers: { + setReasoningOptions(state, action: PayloadAction) { + state.options = action.payload + }, + }, +}) + +const { setReasoningOptions } = reasoningSlice.actions +export default reasoningSlice.reducer + +register('reasoning_options_get', (data, dispatch) => { + const d = data as { success: false } | ({ success: true } & ReasoningOptions) + if (!d.success) return + const { success: _, ...options } = d + dispatch(setReasoningOptions(options)) +}) diff --git a/app/ui_layer/browser/frontend/src/store/uiState/catalog.ts b/app/ui_layer/browser/frontend/src/store/uiState/catalog.ts index 46c85793..096ca650 100644 --- a/app/ui_layer/browser/frontend/src/store/uiState/catalog.ts +++ b/app/ui_layer/browser/frontend/src/store/uiState/catalog.ts @@ -1,4 +1,5 @@ -import type { MetricsTimePeriod } from '../../types' +import type { MetricsTimePeriod, ReasoningChoice } from '../../types' +import { REASONING_CHOICES } from '../../types' import type { DashboardLayoutsStorage } from '../../pages/Dashboard/layout/types' import type { SettingsCategory } from '../../pages/Settings/types' import { defineUiState, defineUiStateFamily, oneOf } from './defineUiState' @@ -61,6 +62,12 @@ export const UI_STATE = { chat: { /** Speech-recognition language code; '' follows the browser language. */ micLanguage: defineUiState('chat.micLanguage', '', 'preference'), + /** The draft chat's reasoning choice, carried into the session its first + * message creates; null (untouched: the model's default level) again once + * that session exists. */ + draftReasoningEffort: defineUiState('chat.draftReasoningEffort', null, 'session', { + isValid: oneOf([null, ...REASONING_CHOICES]), + }), /** Sent inputs for ↑/↓ recall, oldest first, shared by all sessions. */ inputHistory: defineUiState('chat.inputHistory', [], 'session'), /** By session id: ids of the expanded "Action steps" chunks. */ diff --git a/app/ui_layer/browser/frontend/src/types/index.ts b/app/ui_layer/browser/frontend/src/types/index.ts index a11de1b6..333946cd 100644 --- a/app/ui_layer/browser/frontend/src/types/index.ts +++ b/app/ui_layer/browser/frontend/src/types/index.ts @@ -51,6 +51,22 @@ export const QUESTION_DISMISSED = '__dismissed__' export type SessionType = 'main' | 'chat' | 'agent_app' +/** How hard the model reasons, picked per chat session in the chat input + * (agent_core/core/models/reasoning.py ReasoningChoice). */ +export type ReasoningChoice = + | 'provider_default' + | 'off' + | 'minimal' + | 'low' + | 'medium' + | 'high' + | 'xhigh' + | 'max' + +export const REASONING_CHOICES: readonly ReasoningChoice[] = [ + 'provider_default', 'off', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', +] + export interface SessionInfo { id: string type: SessionType @@ -58,6 +74,9 @@ export interface SessionInfo { createdAt: string lastActiveAt: string agentAppProjectId?: string | null + /** Always set by the backend; null only on the client-side placeholder for + * the main session before the backend has sent it. */ + reasoningEffort: ReasoningChoice | null } // ───────────────────────────────────────────────────────────────────── diff --git a/app/ui_layer/controller/ui_controller.py b/app/ui_layer/controller/ui_controller.py index 0bd0feb2..3c500a49 100644 --- a/app/ui_layer/controller/ui_controller.py +++ b/app/ui_layer/controller/ui_controller.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Optional +from agent_core.core.models.reasoning import ReasoningChoice from agent_core.utils.logger import logger from app.ui_layer.events.event_bus import EventBus from app.ui_layer.events.event_types import UIEvent, UIEventType @@ -428,8 +429,17 @@ async def handle_option_click(self, value: str, session_id: str) -> None: elif value == "abort_limit": await self._agent.handle_limit_abort(session_id) - async def handle_prompt_enhance(self, user_message: str) -> str: - return await self._agent._handle_prompt_enhance(user_message=user_message) + async def handle_prompt_enhance( + self, + user_message: str, + session_id: Optional[str], + reasoning_choice: Optional[ReasoningChoice], + ) -> str: + return await self._agent._handle_prompt_enhance( + user_message=user_message, + session_id=session_id, + reasoning_choice=reasoning_choice, + ) # ───────────────────────────────────────────────────────────────────── # Event Processing diff --git a/scripts/probe_reasoning.py b/scripts/probe_reasoning.py index 6588f174..ee1dbb2c 100644 --- a/scripts/probe_reasoning.py +++ b/scripts/probe_reasoning.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- -"""Live check that per-model reasoning defaults are accepted by the providers. +"""Live check that per-model reasoning settings are accepted by the providers. The unit and golden tests prove what CraftBot SENDS; only the provider can say whether it ACCEPTS it. This script sends one tiny JSON request per model through the real LLMInterface (same transports, same output cap as the app) -and reports, per model, the reasoning default that was applied and whether -the provider answered or rejected it. +from inside a probe chat session holding the chosen reasoning choice (the +same session binding a real chat uses), and reports, per model, what was +sent and whether the provider answered or rejected it. Every probe is a real, billed API call (a few hundred tokens each, more for models that think). Nothing is sent with --dry-run. @@ -14,6 +15,7 @@ python scripts/probe_reasoning.py # the configured LLM model python scripts/probe_reasoning.py --model openai/gpt-5.2 --model anthropic/claude-sonnet-4-6 python scripts/probe_reasoning.py --all-rows # every table row with credentials + python scripts/probe_reasoning.py --all-rows --choice off # a session choice python scripts/probe_reasoning.py --all-rows --dry-run Exit status is 1 when any probe was rejected, else 0. @@ -44,6 +46,9 @@ #: Table surface -> the provider whose interface serves it. SUBSCRIPTION_SURFACE = "openai_subscription" +#: Id of the in-memory chat session the probes run in (never persisted). +PROBE_SESSION_ID = "reasoning-probe" + @dataclass class ProbeResult: @@ -174,6 +179,10 @@ def _probe( def main(argv: Optional[List[str]] = None) -> int: + from agent_core.core.models.reasoning import ReasoningChoice + from agent_core.core.session.session import Session + from agent_core.core.state.session import StateSession + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument( "--model", @@ -189,10 +198,22 @@ def main(argv: Optional[List[str]] = None) -> int: action="store_true", help="probe every reasoning-table row whose provider has credentials", ) + parser.add_argument( + "--choice", + type=ReasoningChoice, + default=None, + choices=list(ReasoningChoice), + metavar="CHOICE", + help=( + "the probe session's reasoning choice " + f"({', '.join(choice.value for choice in ReasoningChoice)}); " + "default: each model's default level" + ), + ) parser.add_argument( "--dry-run", action="store_true", - help="print the reasoning default per model without calling any API", + help="print what each model would be sent without calling any API", ) args = parser.parse_args(argv) @@ -205,16 +226,27 @@ def main(argv: Optional[List[str]] = None) -> int: plan = [(None, provider, model)] results: List[ProbeResult] = [] - for surface, provider, model in plan: - result = _probe(provider, model, surface, args.dry_run) - if result is None: - continue - results.append(result) - print( - f"{result.status:9s} {result.target:55s} [{result.route}] " - f"{result.decision} :: {result.detail}", - flush=True, - ) + StateSession.start( + PROBE_SESSION_ID, + current_session=Session( + id=PROBE_SESSION_ID, + reasoning_effort=args.choice.value if args.choice else None, + ), + ) + try: + with StateSession.bind(PROBE_SESSION_ID): + for surface, provider, model in plan: + result = _probe(provider, model, surface, args.dry_run) + if result is None: + continue + results.append(result) + print( + f"{result.status:9s} {result.target:55s} [{result.route}] " + f"{result.decision} :: {result.detail}", + flush=True, + ) + finally: + StateSession.end(PROBE_SESSION_ID) rejected = [r for r in results if r.status == "REJECTED"] ok = sum(r.status == "OK" for r in results) diff --git a/tests/llm/test_reasoning_rules.py b/tests/llm/test_reasoning_rules.py index 8a45f976..a375422c 100644 --- a/tests/llm/test_reasoning_rules.py +++ b/tests/llm/test_reasoning_rules.py @@ -1,44 +1,64 @@ # -*- coding: utf-8 -*- -"""Per-model reasoning defaults (agent_core/core/models/reasoning.py). +"""Per-model reasoning, chosen per chat session (agent_core/core/models/reasoning.py). -Four layers: +Five layers: 1. The table: every row is well formed, uses only the levels its wire can - carry, and is filed under a provider whose transport can send it. -2. The level policy: one level below the strongest, never above "high", - and never below the provider's own default (then the strongest level). + carry, and renders every possible choice through its provider's + transport without error. +2. The choice policy: the default level (one level below the strongest, + never above "high", never below the provider's own default), where new + sessions start, pi-style clamping of a + choice the model lacks, provider default, off, and the temperature and + output-cap consequences of each. 3. Wire rendering: each decision becomes exactly its provider's fields. -4. The transports at CraftBot's real settings (max_tokens 8000): a model with - a rule gets its fields, output cap, and temperature handling; a model - without one is untouched (the golden unruled_* snapshots additionally pin - those payloads byte for byte). +4. The transports at CraftBot's real settings (max_tokens 8000), driven by a + bound session's choice or a per-call choice; a model without a rule is + untouched (the golden unruled_* snapshots additionally pin those payloads + byte for byte). +5. Session plumbing: the stored choice, the session binding through sync + actions, and the picker's WebSocket handlers. """ from __future__ import annotations +import asyncio +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List + import pytest +from agent_core.core.impl.action.executor import _atomic_action_internal_async from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.interface import LLMContextOverflowError -from agent_core.core.impl.llm.reasoning_wire import TRANSPORT_WIRES +from agent_core.core.impl.session.manager import SessionManager from agent_core.core.llm.google_gemini_client import _thinking_config from agent_core.core.models.chatgpt_subscription_client import _translate_request from agent_core.core.models.reasoning import ( BUDGET_RUNGS, BUDGET_WIRES, - KNOWN_LEVELS, + CHOICE_LADDER, LEVEL_CEILING, REASONING_RULES, + ReasoningChoice, ReasoningDecision, ReasoningRule, ReasoningWire, + clamp_choice, + reasoning_options, resolve_reasoning, - target_level, + default_choice, + default_level, ) from agent_core.core.models.registry import get_registry +from agent_core.core.session.session import Session +from agent_core.core.state.session import StateSession from .golden.conftest import GOLDEN_SYSTEM_PROMPT, build_interface +C = ReasoningChoice + #: The Anthropic SDK refuses non-streaming requests above this max_tokens. ANTHROPIC_NON_STREAMING_CEILING = 21_333 #: Anthropic and OpenRouter reject thinking budgets below this. @@ -55,6 +75,17 @@ def _auth_mode(surface: str) -> str: return "subscription" if surface == "openai_subscription" else "api_key" +def _resolve_default(provider, model, auth_mode="api_key"): + """The decision for a session holding the model's default choice.""" + return resolve_reasoning( + provider, model, auth_mode, default_choice(provider, model, auth_mode) + ) + + +#: Every reasoning level, weakest first (the ladder without "off"). +LEVELS = tuple(choice.value for choice in CHOICE_LADDER[1:]) + + ROWS = [ (surface, model, rule) for surface, rows in REASONING_RULES.items() @@ -62,6 +93,13 @@ def _auth_mode(surface: str) -> str: ] ROW_IDS = [f"{surface}/{model}" for surface, model, _ in ROWS] +_RENDERERS = { + "chat_completions": reasoning_wire.chat_completions_fields, + "anthropic_messages": reasoning_wire.anthropic_fields, + "bedrock_converse": reasoning_wire.anthropic_fields, + "gemini_native": reasoning_wire.gemini_thinking_kwargs, +} + # ─────────────────────────────── 1. the table ─────────────────────────────── @@ -71,31 +109,41 @@ def test_row_is_well_formed(surface, model, rule): assert model == model.strip() and model assert rule.levels, "a rule needs at least one level" assert len(set(rule.levels)) == len(rule.levels) - assert all(level in KNOWN_LEVELS for level in rule.levels) - assert list(rule.levels) == sorted(rule.levels, key=KNOWN_LEVELS.index) + assert all(level in LEVELS for level in rule.levels) + assert list(rule.levels) == sorted(rule.levels, key=LEVELS.index) assert rule.provider_default is None or rule.provider_default in rule.levels assert rule.source.startswith("https://") - assert rule.output_tokens >= 0 + assert rule.output_tokens >= 0 and rule.extended_output_tokens >= 0 + if rule.requires_level: + assert rule.provider_default is not None, "a required level needs a default" if rule.wire in BUDGET_WIRES: assert rule.levels == BUDGET_RUNGS assert len(rule.budgets) == len(rule.levels) assert list(rule.budgets) == sorted(set(rule.budgets)) assert rule.budgets[0] >= MIN_THINKING_BUDGET + for level in rule.levels: + # Every rung leaves room for an answer under its own cap. + assert rule.budget_for(level) < rule.output_tokens_for(level) else: assert rule.budgets == () @pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) -def test_row_is_sendable_by_its_providers_transport(surface, model, rule): - profile = get_registry().get(_provider(surface)) - assert profile is not None, f"{surface!r} is not a registered provider" - assert rule.wire in TRANSPORT_WIRES[profile.wire] +def test_every_choice_renders_on_every_row(surface, model, rule): + render = _RENDERERS[get_registry().get(_provider(surface)).wire] + for choice in ReasoningChoice: + decision = resolve_reasoning( + _provider(surface), model, _auth_mode(surface), choice + ) + assert decision.requested is choice + render(decision) # raises if the row asks for something unsendable @pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) def test_anthropic_caps_stay_below_the_sdk_non_streaming_ceiling(surface, model, rule): if rule.wire in (ReasoningWire.ANTHROPIC_ADAPTIVE, ReasoningWire.ANTHROPIC_BUDGET): - assert 0 < rule.output_tokens < ANTHROPIC_NON_STREAMING_CEILING + for level in rule.levels: + assert 0 < rule.output_tokens_for(level) < ANTHROPIC_NON_STREAMING_CEILING def test_duplicate_model_ids_are_refused(): @@ -124,34 +172,37 @@ def test_duplicate_model_ids_are_refused(): (("low", "high", "max"), "high", "high"), (("high",), None, "high"), (BUDGET_RUNGS, None, "high"), + # minimal is selectable but never the default, nor counted for it. + (("minimal", "low", "medium", "high"), "medium", "medium"), + (("minimal", "low", "medium", "high"), "minimal", "medium"), ], ) -def test_target_level_policy(levels, provider_default, expected): +def test_default_level_policy(levels, provider_default, expected): rule = ReasoningRule( wire=ReasoningWire.EFFORT, levels=levels, provider_default=provider_default, source="https://example.test", ) - assert target_level(rule) == expected + assert default_level(rule) == expected @pytest.mark.parametrize("surface, model, rule", ROWS, ids=ROW_IDS) -def test_every_row_resolves_within_the_policy(surface, model, rule): - decision = resolve_reasoning(_provider(surface), model, _auth_mode(surface)) +def test_default_resolves_within_the_policy_on_every_row(surface, model, rule): + decision = _resolve_default(_provider(surface), model, _auth_mode(surface)) assert decision is not None assert decision.key == f"{surface}/{model}" - chosen = rule.levels.index(decision.level) - if rule.provider_default is not None: + assert decision.choice is decision.requested and not decision.off + assert decision.requested is C(decision.level) + ladder = [level for level in rule.levels if level != "minimal"] + chosen = ladder.index(decision.level) + if rule.provider_default in ladder: # Never weaker than what the provider applies when it is omitted. - assert chosen >= rule.levels.index(rule.provider_default) - if LEVEL_CEILING in rule.levels and chosen > rule.levels.index(LEVEL_CEILING): + assert chosen >= ladder.index(rule.provider_default) + if LEVEL_CEILING in ladder and chosen > ladder.index(LEVEL_CEILING): # Above the ceiling only when the provider's own default already is. - assert rule.provider_default is not None - assert rule.levels.index(rule.provider_default) > rule.levels.index( - LEVEL_CEILING - ) - assert chosen == len(rule.levels) - 1 + assert ladder.index(rule.provider_default) > ladder.index(LEVEL_CEILING) + assert chosen == len(ladder) - 1 if decision.budget_tokens is not None: assert decision.budget_tokens < decision.output_tokens @@ -183,8 +234,8 @@ def test_every_row_resolves_within_the_policy(surface, model, rule): ("cerebras", "gpt-oss-120b", "api_key", "medium"), ], ) -def test_resolved_levels(provider, model, auth_mode, expected): - assert resolve_reasoning(provider, model, auth_mode).level == expected +def test_default_levels(provider, model, auth_mode, expected): + assert _resolve_default(provider, model, auth_mode).level == expected @pytest.mark.parametrize( @@ -197,18 +248,105 @@ def test_resolved_levels(provider, model, auth_mode, expected): ("gemini", "gemini-2.5-flash", 18_432), ], ) -def test_budget_models_resolve_to_the_high_rung(provider, model, budget): - decision = resolve_reasoning(provider, model) +def test_budget_models_default_to_the_high_rung(provider, model, budget): + decision = _resolve_default(provider, model) assert decision.level == "high" assert decision.budget_tokens == budget +@pytest.mark.parametrize( + "provider, model, requested, effective", + [ + # pi's clampThinkingLevel: nearest available at or above, else below. + ("openai", "gpt-5.2", C.MAX, C.XHIGH), + ("openai", "gpt-5.2", C.MINIMAL, C.LOW), + ("openai", "gpt-5.2", C.OFF, C.OFF), + ("openai", "gpt-5", C.OFF, C.MINIMAL), + ("openai", "gpt-5", C.XHIGH, C.HIGH), + ("openai", "gpt-6.1-sol", C.OFF, C.LOW), + ("anthropic", "claude-sonnet-4-5", C.XHIGH, C.MAX), + ("anthropic", "claude-opus-5-5", C.OFF, C.LOW), + ("gemini", "gemini-3.1-pro-preview", C.MINIMAL, C.LOW), + ("gemini", "gemini-2.5-pro", C.OFF, C.LOW), + ("glm", "glm-5.2", C.LOW, C.HIGH), + ("grok", "grok-4.5", C.OFF, C.LOW), + ("grok", "grok-4.5", C.XHIGH, C.HIGH), + # Meta-choices are never clamped, except provider_default on a model + # whose default is no reasoning, where it IS off. + ("openai", "gpt-5.5", C.PROVIDER_DEFAULT, C.PROVIDER_DEFAULT), + ("openai", "gpt-5.2", C.PROVIDER_DEFAULT, C.OFF), + ], +) +def test_choices_clamp_like_pi(provider, model, requested, effective): + decision = resolve_reasoning(provider, model, "api_key", requested) + assert decision.requested is requested + assert decision.choice is effective + assert clamp_choice(REASONING_RULES[provider][model], requested) is effective + + +def test_off_sends_nothing_where_the_default_is_no_reasoning(): + decision = resolve_reasoning("openai", "gpt-5.2", "api_key", C.OFF) + assert decision.level is None and not decision.off + assert decision.output_tokens == 0 + + +def test_off_sends_the_disable_form_where_the_model_thinks_by_default(): + decision = resolve_reasoning("openai", "gpt-5.5", "api_key", C.OFF) + assert decision.off and decision.level is None + assert decision.output_tokens == 0 + assert not decision.omit_temperature # gpt-5.5 accepts temperature at "none" + + +def test_provider_default_on_a_model_that_thinks_by_default(): + decision = resolve_reasoning("openai", "gpt-5.5", "api_key", C.PROVIDER_DEFAULT) + assert decision.level is None and not decision.off + # It reasons (at the provider's medium), so it needs the room and the + # temperature rule of a reasoning request. + assert decision.output_tokens == 32_000 + assert decision.omit_temperature + + +def test_provider_default_sends_the_documented_level_where_one_is_required(): + decision = resolve_reasoning( + "openai", "gpt-6.1-sol", "subscription", C.PROVIDER_DEFAULT + ) + assert decision.level == "low" + + +def test_claude_47_rejects_temperature_even_without_thinking(): + decision = resolve_reasoning("anthropic", "claude-opus-4-7", "api_key", C.OFF) + assert decision.level is None and not decision.off + assert decision.omit_temperature + + +def test_claude_46_keeps_temperature_when_not_thinking(): + decision = resolve_reasoning("anthropic", "claude-sonnet-4-6", "api_key", C.OFF) + assert not decision.omit_temperature + + +def test_extended_levels_get_the_extended_cap(): + assert ( + resolve_reasoning("openai", "gpt-5.2", "api_key", C.XHIGH).output_tokens + == 64_000 + ) + assert ( + resolve_reasoning("openai", "gpt-5.2", "api_key", C.HIGH).output_tokens + == 32_000 + ) + # Anthropic stays under the SDK's non-streaming ceiling at every level. + assert ( + resolve_reasoning( + "anthropic", "claude-opus-4-8", "api_key", C.MAX + ).output_tokens + == 21_000 + ) + + @pytest.mark.parametrize( "provider, model, auth_mode", [ ("openai", "gpt-4o", "api_key"), ("openai", "gpt-5.2", "subscription"), # not in the Codex catalogue - ("openai", "gpt-5.5", "api_key_typo"), # unknown auth mode: api surface ("openai", "GPT-5.2", "api_key"), # ids are exact ("openai", " gpt-5.2", "api_key"), ("openai", "gpt-5.2-pro", "api_key"), @@ -222,15 +360,30 @@ def test_budget_models_resolve_to_the_high_rung(provider, model, budget): ], ) def test_models_without_a_rule_resolve_to_none(provider, model, auth_mode): - if auth_mode == "api_key_typo": - # A non-subscription auth mode reads the public-API surface. - assert resolve_reasoning(provider, model, auth_mode).key == "openai/gpt-5.5" - return - assert resolve_reasoning(provider, model, auth_mode) is None + for choice in ReasoningChoice: + assert resolve_reasoning(provider, model, auth_mode, choice) is None + assert reasoning_options(provider, model, auth_mode) is None + + +def test_options_offer_what_the_model_accepts(): + gpt52 = reasoning_options("openai", "gpt-5.2-2025-12-11", "api_key") + # Omitting the parameter is off on gpt-5.2: no separate provider default. + assert gpt52.choices == (C.OFF, C.LOW, C.MEDIUM, C.HIGH, C.XHIGH) + assert gpt52.default_level == "high" + assert gpt52.provider_default == "off" + assert set(gpt52.resolution) == set(ReasoningChoice) + + gpt55 = reasoning_options("openai", "gpt-5.5", "api_key") + assert gpt55.choices[:2] == (C.PROVIDER_DEFAULT, C.OFF) + assert gpt55.provider_default == "medium" + + pro = reasoning_options("gemini", "gemini-2.5-pro", "api_key") + assert pro.choices == (C.PROVIDER_DEFAULT, C.LOW, C.MEDIUM, C.HIGH, C.MAX) + assert pro.provider_default == "dynamic" def test_output_cap_never_lowers_the_callers_cap(): - decision = resolve_reasoning("openai", "gpt-5.2") + decision = _resolve_default("openai", "gpt-5.2") assert decision.output_cap(APP_MAX_TOKENS) == 32_000 assert decision.output_cap(50_000) == 50_000 @@ -238,59 +391,97 @@ def test_output_cap_never_lowers_the_callers_cap(): # ─────────────────────────────── 3. rendering ─────────────────────────────── -def _decision(wire, level="high", budget=None): +def _decision(wire, level="high", budget=None, off=False): return ReasoningDecision( key="test/model", + requested=C.HIGH, + choice=C.OFF if off else C.HIGH, wire=wire, level=level, budget_tokens=budget, + off=off, output_tokens=0, omit_temperature=False, ) def test_chat_completions_fields(): - assert reasoning_wire.chat_completions_fields(_decision(ReasoningWire.EFFORT)) == ( - {"reasoning_effort": "high"}, + render = reasoning_wire.chat_completions_fields + assert render(_decision(ReasoningWire.EFFORT)) == ({"reasoning_effort": "high"}, {}) + assert render(_decision(ReasoningWire.EFFORT, level=None, off=True)) == ( + {"reasoning_effort": "none"}, + {}, + ) + assert render(_decision(ReasoningWire.EFFORT, level=None)) == ({}, {}) + assert render(_decision(ReasoningWire.OPENROUTER_EFFORT)) == ( + {}, + {"reasoning": {"effort": "high"}}, + ) + assert render(_decision(ReasoningWire.OPENROUTER_EFFORT, level=None, off=True)) == ( + {}, + {"reasoning": {"effort": "none"}}, + ) + assert render(_decision(ReasoningWire.OPENROUTER_BUDGET, budget=12_288)) == ( {}, + {"reasoning": {"max_tokens": 12_288}}, ) - assert reasoning_wire.chat_completions_fields( - _decision(ReasoningWire.OPENROUTER_EFFORT) - ) == ({}, {"reasoning": {"effort": "high"}}) - assert reasoning_wire.chat_completions_fields( - _decision(ReasoningWire.OPENROUTER_BUDGET, budget=12_288) - ) == ({}, {"reasoning": {"max_tokens": 12_288}}) def test_anthropic_fields(): - assert reasoning_wire.anthropic_fields( - _decision(ReasoningWire.ANTHROPIC_ADAPTIVE) - ) == {"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}} - assert reasoning_wire.anthropic_fields( - _decision(ReasoningWire.ANTHROPIC_BUDGET, budget=12_288) - ) == {"thinking": {"type": "enabled", "budget_tokens": 12_288}} + render = reasoning_wire.anthropic_fields + assert render(_decision(ReasoningWire.ANTHROPIC_ADAPTIVE)) == { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "high"}, + } + assert render(_decision(ReasoningWire.ANTHROPIC_BUDGET, budget=12_288)) == { + "thinking": {"type": "enabled", "budget_tokens": 12_288} + } + assert render( + _decision(ReasoningWire.ANTHROPIC_ADAPTIVE, level=None, off=True) + ) == {"thinking": {"type": "disabled"}} + assert render(_decision(ReasoningWire.ANTHROPIC_ADAPTIVE, level=None)) == {} def test_gemini_thinking_kwargs(): - assert reasoning_wire.gemini_thinking_kwargs( - _decision(ReasoningWire.GEMINI_LEVEL, level="medium") - ) == {"thinking_level": "medium"} - assert reasoning_wire.gemini_thinking_kwargs( - _decision(ReasoningWire.GEMINI_BUDGET, budget=24_576) - ) == {"thinking_budget": 24_576} + render = reasoning_wire.gemini_thinking_kwargs + assert render(_decision(ReasoningWire.GEMINI_LEVEL, level="medium")) == { + "thinking_level": "medium" + } + assert render(_decision(ReasoningWire.GEMINI_BUDGET, budget=24_576)) == { + "thinking_budget": 24_576 + } + assert render(_decision(ReasoningWire.GEMINI_BUDGET, level=None, off=True)) == { + "thinking_budget": 0 + } + assert render(_decision(ReasoningWire.GEMINI_LEVEL, level=None)) == {} @pytest.mark.parametrize( - "render, wire", + "render, decision", [ - (reasoning_wire.chat_completions_fields, ReasoningWire.ANTHROPIC_ADAPTIVE), - (reasoning_wire.anthropic_fields, ReasoningWire.EFFORT), - (reasoning_wire.gemini_thinking_kwargs, ReasoningWire.OPENROUTER_EFFORT), + ( + reasoning_wire.chat_completions_fields, + _decision(ReasoningWire.ANTHROPIC_ADAPTIVE), + ), + (reasoning_wire.anthropic_fields, _decision(ReasoningWire.EFFORT)), + ( + reasoning_wire.gemini_thinking_kwargs, + _decision(ReasoningWire.OPENROUTER_EFFORT), + ), + # Wires without a disable form. + ( + reasoning_wire.gemini_thinking_kwargs, + _decision(ReasoningWire.GEMINI_LEVEL, level=None, off=True), + ), + ( + reasoning_wire.chat_completions_fields, + _decision(ReasoningWire.OPENROUTER_BUDGET, level=None, off=True), + ), ], ) -def test_a_wire_the_transport_cannot_send_raises(render, wire): - with pytest.raises(ValueError, match="wrong provider"): - render(_decision(wire)) +def test_what_a_transport_cannot_send_raises(render, decision): + with pytest.raises(ValueError, match="cannot send"): + render(decision) def test_gemini_thinking_config_builder(): @@ -315,6 +506,24 @@ def test_codex_translator_uses_the_resolved_effort(): # ────────────────────── 4. transports at CraftBot's settings ───────────────── +@pytest.fixture() +def bound_session(): + """Register a session with a given reasoning choice and bind to it.""" + registered: List[str] = [] + + def _bind(choice: ReasoningChoice, session_id: str = "s-reasoning-test"): + StateSession.start( + session_id, + current_session=Session(id=session_id, reasoning_effort=choice.value), + ) + registered.append(session_id) + return StateSession.bind(session_id) + + yield _bind + for session_id in registered: + StateSession.end(session_id) + + def _one_call(monkeypatch, provider, model, auth_mode="api_key"): iface, rec = build_interface(monkeypatch, provider, model) iface.max_tokens = APP_MAX_TOKENS @@ -324,7 +533,7 @@ def _one_call(monkeypatch, provider, model, auth_mode="api_key"): return iface, rec.calls[0]["payload"] -def test_openai_ruled_model(monkeypatch): +def test_openai_default_level(monkeypatch): _, payload = _one_call(monkeypatch, "openai", "gpt-5.2-2025-12-11") assert payload["reasoning_effort"] == "high" assert payload["max_completion_tokens"] == 32_000 @@ -332,12 +541,41 @@ def test_openai_ruled_model(monkeypatch): assert payload["response_format"] == {"type": "json_object"} -def test_openai_unruled_model(monkeypatch): - _, payload = _one_call(monkeypatch, "openai", "gpt-4o") +def test_openai_unruled_model(monkeypatch, bound_session): + with bound_session(C.XHIGH): + _, payload = _one_call(monkeypatch, "openai", "gpt-4o") assert "reasoning_effort" not in payload assert payload["max_completion_tokens"] == APP_MAX_TOKENS +def test_bound_session_choice_drives_the_request(monkeypatch, bound_session): + with bound_session(C.XHIGH): + _, payload = _one_call(monkeypatch, "openai", "gpt-5.2-2025-12-11") + assert payload["reasoning_effort"] == "xhigh" + assert payload["max_completion_tokens"] == 64_000 + + +def test_bound_session_off(monkeypatch, bound_session): + with bound_session(C.OFF): + _, payload = _one_call(monkeypatch, "openai", "gpt-5.5") + assert payload["reasoning_effort"] == "none" + assert payload["max_completion_tokens"] == APP_MAX_TOKENS + + +def test_per_call_choice_wins_over_the_bound_session(monkeypatch, bound_session): + iface, rec = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + iface.max_tokens = APP_MAX_TOKENS + with bound_session(C.LOW): + asyncio.run( + iface.generate_response_async( + system_prompt=GOLDEN_SYSTEM_PROMPT, + user_prompt="hi", + reasoning_choice=C.MEDIUM, + ) + ) + assert rec.calls[0]["payload"]["reasoning_effort"] == "medium" + + def test_subscription_request_carries_the_codex_row(monkeypatch): _, payload = _one_call(monkeypatch, "openai", "gpt-5.5", auth_mode="subscription") assert payload["reasoning_effort"] == "high" @@ -363,7 +601,7 @@ def test_cerebras_cap_is_clamped_by_the_profile(monkeypatch): def test_glm_bad_model_is_pinned_at_its_max(monkeypatch): _, payload = _one_call(monkeypatch, "glm", "glm-5.3") assert payload["reasoning_effort"] == "max" - assert payload["max_tokens"] == 32_000 + assert payload["max_tokens"] == 64_000 def test_openrouter_openai_row(monkeypatch): @@ -374,7 +612,7 @@ def test_openrouter_openai_row(monkeypatch): assert payload["max_tokens"] == 32_000 -def test_anthropic_ruled_model(monkeypatch): +def test_anthropic_default_level(monkeypatch): _, payload = _one_call(monkeypatch, "anthropic", "claude-opus-5-5") assert payload["thinking"] == {"type": "adaptive"} assert payload["output_config"] == {"effort": "high"} @@ -382,6 +620,23 @@ def test_anthropic_ruled_model(monkeypatch): assert "extra_body" not in payload # no temperature +def test_anthropic_47_provider_default_still_drops_temperature( + monkeypatch, bound_session +): + with bound_session(C.PROVIDER_DEFAULT): + _, payload = _one_call(monkeypatch, "anthropic", "claude-opus-4-7") + assert "thinking" not in payload and "output_config" not in payload + assert "extra_body" not in payload + assert payload["max_tokens"] == 16_384 + + +def test_anthropic_46_off_keeps_temperature(monkeypatch, bound_session): + with bound_session(C.OFF): + _, payload = _one_call(monkeypatch, "anthropic", "claude-sonnet-4-6") + assert "thinking" not in payload + assert payload["extra_body"] == {"temperature": 0.0} + + def test_anthropic_unruled_model(monkeypatch): _, payload = _one_call(monkeypatch, "anthropic", "claude-3-5-haiku-20241022") assert "thinking" not in payload and "output_config" not in payload @@ -398,6 +653,13 @@ def test_bedrock_adaptive_row(monkeypatch): assert payload["inferenceConfig"] == {"maxTokens": 21_000} +def test_bedrock_off_disables_thinking(monkeypatch, bound_session): + with bound_session(C.OFF): + _, payload = _one_call(monkeypatch, "bedrock", "us.anthropic.claude-opus-5") + assert payload["additionalModelRequestFields"] == {"thinking": {"type": "disabled"}} + assert payload["inferenceConfig"] == {"maxTokens": APP_MAX_TOKENS} + + def test_bedrock_unruled_model(monkeypatch): _, payload = _one_call(monkeypatch, "bedrock", "meta.llama3-3-70b-instruct-v1:0") assert "additionalModelRequestFields" not in payload @@ -414,17 +676,35 @@ def test_gemini_level_row(monkeypatch): assert config["maxOutputTokens"] == 32_768 -def test_gemini_caller_budget_wins_over_the_default(monkeypatch): +def test_gemini_off_is_a_zero_budget(monkeypatch, bound_session): + with bound_session(C.OFF): + _, payload = _one_call(monkeypatch, "gemini", "gemini-2.5-flash") + config = payload["body"]["generationConfig"] + assert config["thinkingConfig"] == {"thinkingBudget": 0} + assert config["maxOutputTokens"] == APP_MAX_TOKENS + + +def test_gemini_provider_default_sends_no_thinking_config(monkeypatch, bound_session): + with bound_session(C.PROVIDER_DEFAULT): + _, payload = _one_call(monkeypatch, "gemini", "gemini-2.5-pro") + config = payload["body"]["generationConfig"] + assert "thinkingConfig" not in config + # 2.5 Pro still thinks dynamically, so the request keeps the room. + assert config["maxOutputTokens"] == 32_768 + + +def test_gemini_caller_budget_wins_over_the_session(monkeypatch, bound_session): iface, rec = build_interface(monkeypatch, "gemini", "gemini-3.5-flash") iface.max_tokens = APP_MAX_TOKENS iface._begin_call(thinking_budget=512) - iface._generate_response_sync(GOLDEN_SYSTEM_PROMPT, "hi") + with bound_session(C.HIGH): + iface._generate_response_sync(GOLDEN_SYSTEM_PROMPT, "hi") config = rec.calls[0]["payload"]["body"]["generationConfig"] assert config["thinkingConfig"] == {"thinkingBudget": 512} assert config["maxOutputTokens"] == APP_MAX_TOKENS -def test_context_check_reserves_the_reasoning_cap(monkeypatch): +def test_context_check_reserves_the_reasoning_cap(monkeypatch, bound_session): import app.config as app_config monkeypatch.setattr(app_config, "get_context_window", lambda: 40_000) @@ -441,9 +721,201 @@ def test_context_check_reserves_the_reasoning_cap(monkeypatch): with pytest.raises(LLMContextOverflowError, match="32000 reserved for output"): ruled._check_context_fits(None, prompt) + # A session that turned reasoning off reserves only the caller's cap. + with bound_session(C.OFF): + assert ruled._output_reservation() == APP_MAX_TOKENS + ruled._check_context_fits(None, prompt) + def test_decision_follows_a_model_switch(monkeypatch): iface, _ = build_interface(monkeypatch, "openai", "gpt-4o") assert iface.reasoning_decision() is None + assert iface.reasoning_options() is None iface.model = "gpt-5.2" assert iface.reasoning_decision().level == "high" + assert iface.reasoning_options().default_level == "high" + + +# ──────────────────────────── 5. session plumbing ─────────────────────────── + + +def test_session_round_trips_its_choice(): + session = Session(id="s1", reasoning_effort=C.XHIGH.value) + assert Session.from_dict(session.to_dict()).reasoning_effort == "xhigh" + # Absent or no longer valid (the retired "auto"): discarded, so the + # session manager reseeds it on restore. + assert Session.from_dict({"id": "s2"}).reasoning_effort is None + assert ( + Session.from_dict({"id": "s3", "reasoning_effort": "auto"}).reasoning_effort + is None + ) + assert ( + Session.from_dict({"id": "s4", "reasoning_effort": "turbo"}).reasoning_effort + is None + ) + + +@pytest.mark.parametrize( + "provider, model, expected", + [ + ("openai", "gpt-5.2-2025-12-11", C.HIGH), + ("groq", "openai/gpt-oss-120b", C.MEDIUM), + ("glm", "glm-5.3", C.MAX), + # No rule: the default ceiling, clamped once a model with a rule runs. + ("openai", "gpt-4o", C.HIGH), + (None, None, C.HIGH), + ], +) +def test_default_choice_a_new_session_starts_at(provider, model, expected): + assert default_choice(provider, model) is expected + + +def test_new_and_restored_sessions_are_seeded_with_the_model_default( + monkeypatch, tmp_path: Path +): + iface, _ = build_interface(monkeypatch, "groq", "openai/gpt-oss-120b") + persisted: List[Dict[str, Any]] = [] + manager = SessionManager( + event_stream_manager=None, + llm_interface=iface, + workspace_root=tmp_path, + on_session_persist=lambda session: persisted.append(session.to_dict()), + ) + created = manager.create_session() + restored = manager.restore_session( + Session.from_dict({"id": "s-restored", "reasoning_effort": "auto"}) + ) + kept = manager.restore_session( + Session.from_dict({"id": "s-kept", "reasoning_effort": "xhigh"}) + ) + try: + assert created.reasoning_effort == "medium" + # The retired value is discarded and reseeded, and the seed is saved. + assert restored.reasoning_effort == "medium" + assert persisted[-1] == { + **persisted[-1], + "id": "s-restored", + "reasoning_effort": "medium", + } + # A valid stored choice is kept as is, even one the model lacks. + assert kept.reasoning_effort == "xhigh" + finally: + for session_id in (created.id, "s-restored", "s-kept"): + StateSession.end(session_id) + + +def test_work_outside_any_session_uses_the_model_default(monkeypatch): + iface, _ = build_interface(monkeypatch, "glm", "glm-5.3") + assert iface.reasoning_choice() is C.MAX + assert iface.reasoning_decision().level == "max" + + +def test_session_manager_creates_and_sets_the_choice(tmp_path: Path): + persisted: List[Dict[str, Any]] = [] + manager = SessionManager( + event_stream_manager=None, + workspace_root=tmp_path, + on_session_persist=lambda session: persisted.append(session.to_dict()), + ) + session = manager.create_session(reasoning_effort=C.LOW) + try: + assert session.reasoning_effort == "low" + assert manager.set_reasoning_effort(session.id, C.OFF) + assert manager.get(session.id).reasoning_effort == "off" + assert persisted[-1]["reasoning_effort"] == "off" + assert not manager.set_reasoning_effort("no-such-session", C.OFF) + finally: + StateSession.end(session.id) + + +def test_sync_actions_run_in_the_callers_session_binding(bound_session): + code = ( + "def probe(input_data):\n" + " from agent_core.core.state.session import StateSession\n" + " state = StateSession.bound()\n" + " return {'session': state.session_id if state else None}\n" + ) + + async def _run(): + with bound_session(C.HIGH, session_id="s-sync-action"): + return await _atomic_action_internal_async("probe", code, {}, "CLI") + + assert asyncio.run(_run()) == {"session": "s-sync-action"} + + +def _adapter_with(agent) -> Any: + from app.ui_layer.adapters.browser_adapter import BrowserAdapter + + adapter = BrowserAdapter.__new__(BrowserAdapter) + adapter._controller = SimpleNamespace(agent=agent) + adapter.sent = [] + adapter.broadcasts = [] + + async def _send_to(ws, message): + adapter.sent.append(message) + + async def _broadcast(message): + adapter.broadcasts.append(message) + + adapter._send_to = _send_to + adapter._broadcast = _broadcast + return adapter + + +def test_reasoning_options_handler(monkeypatch): + iface, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + adapter = _adapter_with(SimpleNamespace(llm=iface)) + asyncio.run(adapter._handle_reasoning_options_get(ws=None)) + data = adapter.sent[0]["data"] + assert data["configurable"] is True + assert data["choices"] == ["off", "low", "medium", "high", "xhigh"] + assert data["defaultLevel"] == "high" + assert data["providerDefault"] == "off" + assert data["resolution"]["max"] == "xhigh" + + unruled, _ = build_interface(monkeypatch, "openai", "gpt-4o") + adapter = _adapter_with(SimpleNamespace(llm=unruled)) + asyncio.run(adapter._handle_reasoning_options_get(ws=None)) + assert adapter.sent[0]["data"] == { + "success": True, + "model": "gpt-4o", + "configurable": False, + } + + +def test_session_reasoning_set_handler(tmp_path: Path): + manager = SessionManager(event_stream_manager=None, workspace_root=tmp_path) + session = manager.create_session() + agent = SimpleNamespace( + session_manager=manager, + set_session_reasoning_effort=manager.set_reasoning_effort, + ) + adapter = _adapter_with(agent) + try: + asyncio.run( + adapter._handle_session_reasoning_set( + {"sessionId": session.id, "reasoningEffort": "minimal"} + ) + ) + assert session.reasoning_effort == "minimal" + assert adapter.broadcasts[-1]["data"]["session"]["reasoningEffort"] == "minimal" + + # An unknown value changes nothing. + asyncio.run( + adapter._handle_session_reasoning_set( + {"sessionId": session.id, "reasoningEffort": "turbo"} + ) + ) + assert session.reasoning_effort == "minimal" + assert len(adapter.broadcasts) == 1 + finally: + StateSession.end(session.id) + + +def test_draft_reasoning_from_a_message(): + from app.ui_layer.adapters.browser_adapter import BrowserAdapter + + assert BrowserAdapter._draft_reasoning({"reasoningEffort": "xhigh"}) is C.XHIGH + assert BrowserAdapter._draft_reasoning({}) is None + with pytest.raises(ValueError): + BrowserAdapter._draft_reasoning({"reasoningEffort": "turbo"}) From 10ffd5de1ffb1a32a0c47835e9437915b8aed9d1 Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 09:27:32 +0900 Subject: [PATCH 08/55] fix max token ignores the reasoning output cap bug --- agent_core/core/impl/llm/interface.py | 8 +++++++- tests/llm/test_reasoning_rules.py | 29 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index 520330cf..ff75639b 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -1223,6 +1223,11 @@ def fits_context( plus the output reservation, against the window less the headroom the summary request needs. On a session's first request there is no provider count yet, so the whole prompt is counted locally. + + The output reservation is the one the pre-send check + (_check_context_fits) applies, including the reasoning output cap of + the session's choice, so the stream folds before that check could + refuse the request. """ from app.config import get_context_window, get_reserve_tokens @@ -1232,7 +1237,8 @@ def fits_context( else: projected = last + count_tokens(pending) return ( - projected + self.max_tokens <= get_context_window() - get_reserve_tokens() + projected + self._output_reservation() + <= get_context_window() - get_reserve_tokens() ) def end_all_session_caches(self, task_id: str) -> None: diff --git a/tests/llm/test_reasoning_rules.py b/tests/llm/test_reasoning_rules.py index a375422c..26448e4a 100644 --- a/tests/llm/test_reasoning_rules.py +++ b/tests/llm/test_reasoning_rules.py @@ -727,6 +727,35 @@ def test_context_check_reserves_the_reasoning_cap(monkeypatch, bound_session): ruled._check_context_fits(None, prompt) +def test_fold_check_reserves_the_same_cap_as_the_pre_send_check( + monkeypatch, bound_session +): + import app.config as app_config + + # window 128,000 - reserve 16,384 - the output reservation. + monkeypatch.setattr(app_config, "get_context_window", lambda: 128_000) + monkeypatch.setattr(app_config, "get_reserve_tokens", lambda: 16_384) + + def assert_folds_after(iface, last_input_tokens: int) -> None: + key = "task:action_selection" + iface._last_input_tokens[key] = last_input_tokens + assert iface.fits_context("task", "action_selection", "s", "") + iface._last_input_tokens[key] = last_input_tokens + 1 + assert not iface.fits_context("task", "action_selection", "s", "") + + unruled, _ = build_interface(monkeypatch, "openai", "gpt-4o") + unruled.max_tokens = APP_MAX_TOKENS + assert_folds_after(unruled, 103_616) # the caller's 8,000 + + ruled, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + ruled.max_tokens = APP_MAX_TOKENS + assert_folds_after(ruled, 79_616) # default high: 32,000 + with bound_session(C.XHIGH): + assert_folds_after(ruled, 47_616) # 64,000 + with bound_session(C.OFF): + assert_folds_after(ruled, 103_616) # no reasoning: the caller's 8,000 + + def test_decision_follows_a_model_switch(monkeypatch): iface, _ = build_interface(monkeypatch, "openai", "gpt-4o") assert iface.reasoning_decision() is None From 0c054e18b538b2e394a923e5cf3acb4ae604dbeb Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 10:00:12 +0900 Subject: [PATCH 09/55] openai setting update --- .../models/chatgpt_subscription_client.py | 17 +++--- agent_core/core/models/factory.py | 23 +++----- agent_core/core/models/provider_config.py | 20 +++++-- agent_core/core/models/reasoning.py | 21 +++---- agent_file_system/AGENT.md | 2 +- app/data/agent_file_system_template/AGENT.md | 2 +- craftos_integrations/llm_oauth/chatgpt.py | 29 ---------- tests/integrations/test_import_surface.py | 1 - tests/llm/test_reasoning_rules.py | 56 ++++++++++++++++--- 9 files changed, 91 insertions(+), 80 deletions(-) diff --git a/agent_core/core/models/chatgpt_subscription_client.py b/agent_core/core/models/chatgpt_subscription_client.py index c2e82fef..520470b2 100644 --- a/agent_core/core/models/chatgpt_subscription_client.py +++ b/agent_core/core/models/chatgpt_subscription_client.py @@ -163,22 +163,23 @@ def __init__( # silently honoring "best-effort" semantics is fine for fields that just # don't apply at this backend (e.g. ``max_tokens`` becomes "let the # server decide" rather than a hard failure). -#: Effort for a model the reasoning table has no row for. The Codex backend -#: requires a ``reasoning`` block on every request, so unlike every other -#: provider the parameter cannot simply be left out; "medium" is the Codex -#: CLI default and what this translator always sent before per-model rules. -_CODEX_UNRULED_EFFORT = "medium" +#: Effort for a request that carries no ``reasoning_effort``. LLM requests +#: always carry one (every subscription model has a reasoning row); VLM +#: requests send none. The Codex backend requires a ``reasoning`` block on +#: every request, so unlike every other provider the parameter cannot simply +#: be left out; "medium" is the Codex CLI default. +_CODEX_DEFAULT_EFFORT = "medium" def _codex_reasoning_config(requested_effort: Any) -> Dict[str, str]: """Build the ``reasoning`` block Codex requires on every request. ``requested_effort`` is the caller's Chat-Completions - ``reasoning_effort``: the per-model default the chat_completions - transport resolved, or None when the model has no rule. + ``reasoning_effort``: the level the chat_completions transport resolved + for the session's reasoning choice, or None when the caller sends none. ``"auto"`` summary matches the Codex CLI. """ - effort = requested_effort if requested_effort else _CODEX_UNRULED_EFFORT + effort = requested_effort if requested_effort else _CODEX_DEFAULT_EFFORT return {"effort": str(effort), "summary": "auto"} diff --git a/agent_core/core/models/factory.py b/agent_core/core/models/factory.py index e360c96b..7d8162f8 100644 --- a/agent_core/core/models/factory.py +++ b/agent_core/core/models/factory.py @@ -353,25 +353,18 @@ def create( access_token, sub_base_url, extra_headers = oauth - # Codex's accepted-model list lives in the ChatGPT OAuth - # backend module so provider-specific knowledge stays - # colocated with the flow that authenticates against it. - # See ``llm_oauth.chatgpt.CODEX_ACCEPTED_MODELS`` for the - # source-of-truth list and the reasoning behind the fallback. - from craftos_integrations.llm_oauth.chatgpt import ( - CODEX_ACCEPTED_MODELS, - effective_model_for_subscription, - ) - - effective_model, was_substituted = effective_model_for_subscription( - model - ) - if was_substituted: + # The Codex backend serves only the profile's + # subscription_models; any other model runs as the + # profile's subscription default. + if model in cfg.subscription_models: + effective_model = model + else: + effective_model = cfg.subscription_default_model logger.warning( f"[FACTORY] ChatGPT subscription mode rejects model " f"{model!r}; substituting {effective_model!r}. " f"Valid Codex-subscription models: " - f"{sorted(CODEX_ACCEPTED_MODELS)}. Set the model in " + f"{list(cfg.subscription_models)}. Set the model in " f"Settings to silence this warning." ) diff --git a/agent_core/core/models/provider_config.py b/agent_core/core/models/provider_config.py index 51fdaba8..2be66cff 100644 --- a/agent_core/core/models/provider_config.py +++ b/agent_core/core/models/provider_config.py @@ -179,14 +179,24 @@ def resolve_temperature(profile: Optional[ProviderProfile], caller_temperature): settings_key="openai", oauth_backend="chatgpt", subscription_label="Sign in with ChatGPT", - # Codex-accepted models for ChatGPT subscription auth. + # The only models ChatGPT-subscription auth runs: the Codex catalogue's + # listed models, in its order + # (https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json). + # The factory runs any other model as subscription_default_model. + # Codex requires a reasoning level on every request, so each of + # these has a row in agent_core/core/models/reasoning.py + # (tests/llm/test_reasoning_rules.py checks they agree). subscription_models=( - "gpt-5.4", + "gpt-6.1-sol", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", "gpt-5.5", - "gpt-5.4-mini", - "gpt-5.3-codex-spark", ), - subscription_default_model="gpt-5.4", + subscription_default_model="gpt-6.1-sol", supports_prompt_cache_key=True, # OpenAI deprecated `max_tokens` in favor of `max_completion_tokens`; # every current chat model accepts the new field, and reasoning diff --git a/agent_core/core/models/reasoning.py b/agent_core/core/models/reasoning.py index 8d6d5baf..40cb97ff 100644 --- a/agent_core/core/models/reasoning.py +++ b/agent_core/core/models/reasoning.py @@ -466,12 +466,12 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: # ── OpenAI public API (Chat Completions ``reasoning_effort``) ── # Values and defaults: https://developers.openai.com/api/docs/models/. -# Models before gpt-5.1 reject "none" and never accept temperature; gpt-5.1+ -# reject "minimal", default to no reasoning through gpt-5.4, and accept -# temperature only at "none". xhigh arrived with gpt-5.2. "max" exists only on -# the Responses API: Chat Completions rejects it on gpt-5.6 and GPT-6 ("Supported -# values are: 'none', 'low', 'medium', 'high', and 'xhigh'", live API error, -# 2026-10-02; GPT-6 Astra and 6.1 Sol list the same without 'none'). +# Models before gpt-5.1 reject "none"; gpt-5.1+ reject "minimal" and default +# to no reasoning through gpt-5.4. xhigh arrived with gpt-5.2. "max" exists +# only on the Responses API: Chat Completions rejects it on gpt-5.6 and GPT-6 +# ("Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'", live +# API error, 2026-10-02; GPT-6 Astra and 6.1 Sol list the same without 'none'). +# Temperature is the OpenAI profile's: it omits it on every request. _OPENAI_DOCS = "https://developers.openai.com/api/docs/models" @@ -481,7 +481,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="medium", source=f"{_OPENAI_DOCS}/gpt-5", output_tokens=_EFFORT_OUTPUT_TOKENS, - temperature=TemperaturePolicy.OMIT_ALWAYS, ) _OPENAI_O_SERIES = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -489,7 +488,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: provider_default="medium", source="https://developers.openai.com/api/docs/guides/reasoning", output_tokens=_EFFORT_OUTPUT_TOKENS, - temperature=TemperaturePolicy.OMIT_ALWAYS, ) _OPENAI_GPT51 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -498,7 +496,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: source=f"{_OPENAI_DOCS}/gpt-5.1", output_tokens=_EFFORT_OUTPUT_TOKENS, off=ReasoningOff.OMIT, - temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT52_TO_54 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -508,7 +505,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: output_tokens=_EFFORT_OUTPUT_TOKENS, extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, off=ReasoningOff.OMIT, - temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT55 = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -518,7 +514,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: output_tokens=_EFFORT_OUTPUT_TOKENS, extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, off=ReasoningOff.EXPLICIT, - temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) _OPENAI_GPT56_PLUS = ReasoningRule( wire=ReasoningWire.EFFORT, @@ -528,7 +523,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: output_tokens=_EFFORT_OUTPUT_TOKENS, extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, off=ReasoningOff.EXPLICIT, - temperature=TemperaturePolicy.OMIT_WHILE_THINKING, ) #: GPT-6 Astra and GPT-6.1 Sol reject "none" and always reason. _OPENAI_GPT6_ALWAYS_REASONING = ReasoningRule( @@ -538,7 +532,6 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: source=f"{_OPENAI_DOCS}/gpt-6-astra", output_tokens=_EFFORT_OUTPUT_TOKENS, extended_output_tokens=_EFFORT_EXTENDED_OUTPUT_TOKENS, - temperature=TemperaturePolicy.OMIT_ALWAYS, ) _OPENAI_GPT61_SOL = replace( _OPENAI_GPT6_ALWAYS_REASONING, @@ -552,6 +545,8 @@ def _bedrock_ids(base_id: str, *profile_prefixes: str) -> Tuple[str, ...]: # supported_reasoning_levels minus the client-side "ultra" alias; no model # lists "none", so none can be turned off # (https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json). +# The rows are exactly the OpenAI profile's subscription_models, the only +# models subscription auth runs. _CODEX_CATALOG = ( "https://github.com/openai/codex/blob/main/codex-rs/models-manager/models.json" diff --git a/agent_file_system/AGENT.md b/agent_file_system/AGENT.md index 7698f9e3..2036db75 100644 --- a/agent_file_system/AGENT.md +++ b/agent_file_system/AGENT.md @@ -3078,7 +3078,7 @@ Users can authenticate OpenAI or Grok by signing in to their paid subscription ( ChatGPT subscription specifics: - Requests route through OpenAI's Codex backend. CraftBot's JSON-mode action decisions work transparently; only native tool-calls (`tools=[...]`) and streaming are unsupported — neither is CraftBot's normal path, so actions run fine. -- Codex accepts a fixed model set (gpt-5.5, gpt-5.4, gpt-5.4-mini, gpt-5.3-codex-spark; default gpt-5.4); any other model name is silently substituted. +- Codex accepts a fixed model set (gpt-6.1-sol, gpt-6-astra, gpt-6-sol, gpt-6-luna, gpt-5.6-sol, gpt-5.6-terra, gpt-5.6-luna, gpt-5.5; default gpt-6.1-sol); any other model name is silently substituted. - The real hard failure is a Free-tier account (no Plus/Pro/Team): `CHATGPT_SUBSCRIPTION_REJECTED`. That's the "upgrade or switch to an API key" case — do not retry. - If the credential is disconnected mid-session, the client raises an actionable error telling the user to re-save model settings or reconnect. diff --git a/app/data/agent_file_system_template/AGENT.md b/app/data/agent_file_system_template/AGENT.md index 7698f9e3..2036db75 100644 --- a/app/data/agent_file_system_template/AGENT.md +++ b/app/data/agent_file_system_template/AGENT.md @@ -3078,7 +3078,7 @@ Users can authenticate OpenAI or Grok by signing in to their paid subscription ( ChatGPT subscription specifics: - Requests route through OpenAI's Codex backend. CraftBot's JSON-mode action decisions work transparently; only native tool-calls (`tools=[...]`) and streaming are unsupported — neither is CraftBot's normal path, so actions run fine. -- Codex accepts a fixed model set (gpt-5.5, gpt-5.4, gpt-5.4-mini, gpt-5.3-codex-spark; default gpt-5.4); any other model name is silently substituted. +- Codex accepts a fixed model set (gpt-6.1-sol, gpt-6-astra, gpt-6-sol, gpt-6-luna, gpt-5.6-sol, gpt-5.6-terra, gpt-5.6-luna, gpt-5.5; default gpt-6.1-sol); any other model name is silently substituted. - The real hard failure is a Free-tier account (no Plus/Pro/Team): `CHATGPT_SUBSCRIPTION_REJECTED`. That's the "upgrade or switch to an API key" case — do not retry. - If the credential is disconnected mid-session, the client raises an actionable error telling the user to re-save model settings or reconnect. diff --git a/craftos_integrations/llm_oauth/chatgpt.py b/craftos_integrations/llm_oauth/chatgpt.py index e509d2cc..82bf36d1 100644 --- a/craftos_integrations/llm_oauth/chatgpt.py +++ b/craftos_integrations/llm_oauth/chatgpt.py @@ -58,35 +58,6 @@ logger = get_logger(__name__) -# ════════════════════════════════════════════════════════════════════════ -# Accepted-model list for ChatGPT-subscription auth -# ════════════════════════════════════════════════════════════════════════ - -CODEX_ACCEPTED_MODELS = frozenset( - { - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.3-codex-spark", - } -) - -CODEX_DEFAULT_MODEL = "gpt-5.4" - - -def effective_model_for_subscription(model: str) -> Tuple[str, bool]: - """Return ``(effective, was_substituted)`` for a Codex-subscription call. - - If ``model`` is one of the accepted names it passes through - unchanged. Otherwise it's replaced with ``CODEX_DEFAULT_MODEL`` and - the second return value is ``True`` so the caller can log the - substitution once. - """ - if model in CODEX_ACCEPTED_MODELS: - return model, False - return CODEX_DEFAULT_MODEL, True - - AUTH_URL = "https://auth.openai.com/oauth/authorize" TOKEN_URL = "https://auth.openai.com/oauth/token" diff --git a/tests/integrations/test_import_surface.py b/tests/integrations/test_import_surface.py index 6088a387..2ea96f2b 100644 --- a/tests/integrations/test_import_surface.py +++ b/tests/integrations/test_import_surface.py @@ -46,7 +46,6 @@ def test_llm_oauth_entry_points(): assert callable(tokens.get_bearer) assert callable(tokens.status) assert callable(chatgpt.load) - assert chatgpt.CODEX_ACCEPTED_MODELS assert callable(grok.load) # Not connected on a clean checkout, but the call must work. diff --git a/tests/llm/test_reasoning_rules.py b/tests/llm/test_reasoning_rules.py index 26448e4a..78abecc8 100644 --- a/tests/llm/test_reasoning_rules.py +++ b/tests/llm/test_reasoning_rules.py @@ -154,6 +154,38 @@ def test_duplicate_model_ids_are_refused(): reasoning._surface({"m": rule}, {"m": rule}) +def test_subscription_rows_are_exactly_the_models_subscription_auth_runs(): + # Codex requires a level on every request, so a subscription model + # without a row would run at Codex's fallback, and a row for a model + # subscription auth never runs would be dead. + profile = get_registry().get("openai") + assert set(REASONING_RULES["openai_subscription"]) == set( + profile.subscription_models + ) + assert profile.subscription_default_model in profile.subscription_models + + +def test_subscription_runs_an_unlisted_model_as_the_default(monkeypatch): + from agent_core.core.models import factory + from agent_core.core.models.types import InterfaceType + + monkeypatch.setattr( + factory, + "_get_oauth_bearer", + lambda provider: ("token", "https://chatgpt.com/backend-api/codex", {}), + ) + for requested, runs in ( + ("gpt-6-sol", "gpt-6-sol"), + ("gpt-5.4", "gpt-6.1-sol"), + ("gpt-5.2-2025-12-11", "gpt-6.1-sol"), + ): + ctx = factory.ModelFactory.create( + provider="openai", interface=InterfaceType.LLM, model_override=requested + ) + assert ctx["auth_mode"] == "subscription" + assert ctx["model"] == runs + + # ─────────────────────────────── 2. the policy ────────────────────────────── @@ -294,16 +326,25 @@ def test_off_sends_the_disable_form_where_the_model_thinks_by_default(): decision = resolve_reasoning("openai", "gpt-5.5", "api_key", C.OFF) assert decision.off and decision.level is None assert decision.output_tokens == 0 - assert not decision.omit_temperature # gpt-5.5 accepts temperature at "none" def test_provider_default_on_a_model_that_thinks_by_default(): decision = resolve_reasoning("openai", "gpt-5.5", "api_key", C.PROVIDER_DEFAULT) assert decision.level is None and not decision.off - # It reasons (at the provider's medium), so it needs the room and the - # temperature rule of a reasoning request. + # It reasons (at the provider's medium), so it needs the room of a + # reasoning request. assert decision.output_tokens == 32_000 - assert decision.omit_temperature + + +def test_temperature_is_dropped_exactly_while_the_request_reasons(): + # OpenRouter forwards an explicit temperature upstream, where reasoning + # rejects it; without reasoning it is accepted. + reasoning = resolve_reasoning( + "openrouter", "openai/gpt-5.5", "api_key", C.PROVIDER_DEFAULT + ) + assert reasoning.level is None and reasoning.omit_temperature + off = resolve_reasoning("openrouter", "openai/gpt-5.5", "api_key", C.OFF) + assert off.off and not off.omit_temperature def test_provider_default_sends_the_documented_level_where_one_is_required(): @@ -498,9 +539,10 @@ def test_codex_translator_uses_the_resolved_effort(): {"model": "gpt-5.5", "messages": messages, "reasoning_effort": "high"}, "k" ) assert ruled["reasoning"] == {"effort": "high", "summary": "auto"} - # No rule: the backend still requires the block, so today's medium stays. - unruled = _translate_request({"model": "gpt-5.4", "messages": messages}, "k") - assert unruled["reasoning"] == {"effort": "medium", "summary": "auto"} + # No reasoning_effort (VLM requests send none): the backend still + # requires the block, so it gets the Codex default. + unset = _translate_request({"model": "gpt-6.1-sol", "messages": messages}, "k") + assert unset["reasoning"] == {"effort": "medium", "summary": "auto"} # ────────────────────── 4. transports at CraftBot's settings ───────────────── From fb3f785edd2e9a5ecf5c88ec950d5174c477f2c1 Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 10:38:57 +0900 Subject: [PATCH 10/55] fix duplicated steps and second decision per request --- agent_core/core/impl/llm/cache/gemini.py | 4 +- agent_core/core/impl/llm/interface.py | 132 ++++++++++-------- agent_core/core/impl/llm/reasoning_wire.py | 70 +++++++++- .../core/impl/llm/transports/__init__.py | 5 +- .../impl/llm/transports/anthropic_messages.py | 29 ++-- .../impl/llm/transports/bedrock_converse.py | 37 +++-- .../impl/llm/transports/byteplus_responses.py | 10 +- .../impl/llm/transports/chat_completions.py | 44 +++--- .../core/impl/llm/transports/gemini_native.py | 35 ++--- tests/llm/test_reasoning_rules.py | 66 ++++++++- tests/test_bedrock_token_normalization.py | 5 +- tests/test_context_budget.py | 4 +- 12 files changed, 291 insertions(+), 150 deletions(-) diff --git a/agent_core/core/impl/llm/cache/gemini.py b/agent_core/core/impl/llm/cache/gemini.py index 1b1376a9..f6779306 100644 --- a/agent_core/core/impl/llm/cache/gemini.py +++ b/agent_core/core/impl/llm/cache/gemini.py @@ -78,7 +78,7 @@ def get_or_create_cache( system_prompt: str, user_prompt: str, call_type: str, - temperature: float, + temperature: Optional[float], max_tokens: int, thinking_budget: Optional[int] = None, thinking_level: Optional[str] = None, @@ -89,7 +89,7 @@ def get_or_create_cache( system_prompt: The system prompt to cache. user_prompt: The user prompt for this request. call_type: Type of LLM call (e.g., "reasoning", "action_selection"). - temperature: Sampling temperature. + temperature: Sampling temperature (None: not sent). max_tokens: Maximum output tokens. thinking_budget: Reasoning token budget (Gemini 2.5), forwarded to every generation call. diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index ff75639b..9e3d9b2c 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -17,7 +17,7 @@ import contextvars import re import time -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set, Tuple from agent_core.decorators import profile, OperationCategory @@ -69,15 +69,6 @@ "_llm_call_ctx", default={} ) -# Per-call metadata (prompt identity + start time) propagated from the public -# entry methods down to the capture chokepoint (_call_log_to_db) without -# threading it through every provider method. asyncio.to_thread copies the -# context into the worker thread, so this survives the sync offload, and each -# asyncio Task / thread gets its own copy so concurrent calls don't clobber. -_llm_call_ctx: contextvars.ContextVar[dict] = contextvars.ContextVar( - "_llm_call_ctx", default={} -) - # Session key of the session call in flight. Set by the public session entry # points and read by _report_usage_async, so the input count the provider @@ -236,9 +227,12 @@ def __init__( # multi-provider outage terminates instead of nesting # primary -> fb -> fb-of-fb recursion. self._is_fallback_instance = False - # (provider, model, auth_mode) whose reasoning default was last - # logged, so the decision is logged once per model, not per call. - self._reasoning_logged_for: Optional[tuple] = None + # (provider, model, auth_mode, choice) combinations whose reasoning + # decision has been logged: each is logged once, not per call, even + # while concurrent sessions alternate between different choices. + self._reasoning_logged: Set[ + Tuple[Optional[str], Optional[str], str, ReasoningChoice] + ] = set() # Defer imports to avoid circular dependency from app.models.factory import ModelFactory @@ -783,17 +777,20 @@ def _check_context_fits( system_prompt: Optional[str], user_prompt: Optional[str] = None, messages: Optional[List[dict]] = None, + *, + reasoning: Optional[ReasoningDecision], ) -> None: """Refuse a request that cannot fit the configured context window. Counts the payload that is about to be sent: the system prompt plus either the single user prompt or every accumulated message. Input and - the output reservation share the window. + the output reservation share the window. ``reasoning`` is the + request's decision, the same one its transport sends. """ from app.config import get_context_window window = get_context_window() - reserved = self._output_reservation() + reserved = self._output_reservation(reasoning) budget = window - reserved total = count_tokens(system_prompt or "") @@ -853,18 +850,18 @@ def reasoning_decision(self) -> Optional[ReasoningDecision]: """What the call in flight sends for reasoning. None means the model has no rule in agent_core/core/models/reasoning.py - and its requests carry no reasoning parameter. Resolved on every call - so a model switch (reinitialize), a fallback interface, or a changed - session choice applies from the next request; logged once per - distinct model and choice. + and its requests carry no reasoning parameter. Each request resolves + it once, just before its context-window check, and hands the same + decision to that check and to its transport; so a model switch + (reinitialize), a fallback interface, or a changed session choice + applies from the next request. Logged once per distinct model and + choice. """ choice = self.reasoning_choice() - decision = resolve_reasoning( - self.provider, self.model, self._auth_mode, choice - ) + decision = resolve_reasoning(self.provider, self.model, self._auth_mode, choice) log_key = (self.provider, self.model, self._auth_mode, choice) - if log_key != self._reasoning_logged_for: - self._reasoning_logged_for = log_key + if log_key not in self._reasoning_logged: + self._reasoning_logged.add(log_key) if decision is None: logger.info( f"[REASONING] {self.provider}/{self.model}: no reasoning rule " @@ -874,17 +871,17 @@ def reasoning_decision(self) -> Optional[ReasoningDecision]: logger.info(f"[REASONING] {decision.key}: {decision.describe()}") return decision - def _output_reservation(self) -> int: - """Output tokens a request reserves out of the context window. + def _output_reservation(self, reasoning: Optional[ReasoningDecision]) -> int: + """Output tokens a request with this reasoning decision reserves out + of the context window. - Requests with a reasoning default carry a larger output cap (reasoning - tokens count against it), so the window check reserves that cap; - otherwise the provider would reject the request for size instead. + A reasoning request carries a larger output cap (reasoning tokens + count against it), so the window check reserves that cap; otherwise + the provider would reject the request for size instead. """ - decision = self.reasoning_decision() - if decision is None: + if reasoning is None: return self.max_tokens - return decision.output_cap(self.max_tokens) + return reasoning.output_cap(self.max_tokens) def _generate_response_sync( self, @@ -931,8 +928,15 @@ def _generate_response_sync( ) if _transport is None: # pragma: no cover raise RuntimeError(f"Unknown provider {self.provider!r}") - self._check_context_fits(system_prompt, user_prompt) - response = _transport(self, system_prompt, user_prompt, json_mode=json_mode) + reasoning = self.reasoning_decision() + self._check_context_fits(system_prompt, user_prompt, reasoning=reasoning) + response = _transport( + self, + system_prompt, + user_prompt, + json_mode=json_mode, + reasoning=reasoning, + ) content = response.get("content", "").strip() # Check if response is empty and provide diagnostics @@ -1237,7 +1241,7 @@ def fits_context( else: projected = last + count_tokens(pending) return ( - projected + self._output_reservation() + projected + self._output_reservation(self.reasoning_decision()) <= get_context_window() - get_reserve_tokens() ) @@ -1483,12 +1487,16 @@ def _generate_response_with_session_sync( f"sending {len(contents)} total contents" ) - self._check_context_fits(effective_system_prompt, messages=contents) + reasoning = self.reasoning_decision() + self._check_context_fits( + effective_system_prompt, messages=contents, reasoning=reasoning + ) response = self._generate_gemini( effective_system_prompt, user_prompt, call_type=call_type, contents_override=contents, + reasoning=reasoning, ) assistant_content = response.get("content", "") @@ -1545,12 +1553,16 @@ def _generate_response_with_session_sync( f"{len(history)} history msgs, sending {len(or_messages)} total" ) - self._check_context_fits(None, messages=or_messages) + reasoning = self.reasoning_decision() + self._check_context_fits( + None, messages=or_messages, reasoning=reasoning + ) response = self._generate_openai( effective_system_prompt, user_prompt, call_type=call_type, messages_override=or_messages, + reasoning=reasoning, ) assistant_content = response.get("content", "") @@ -1587,12 +1599,16 @@ def _generate_response_with_session_sync( f"{len(history)} history msgs, sending {len(oa_messages)} total" ) - self._check_context_fits(None, messages=oa_messages) + reasoning = self.reasoning_decision() + self._check_context_fits( + None, messages=oa_messages, reasoning=reasoning + ) response = self._generate_openai( effective_system_prompt, user_prompt, call_type=call_type, messages_override=oa_messages, + reasoning=reasoning, ) assistant_content = response.get("content", "") @@ -1667,12 +1683,16 @@ def _generate_response_with_session_sync( ) # Call Anthropic with the full multi-turn messages - self._check_context_fits(effective_system_prompt, messages=messages) + reasoning = self.reasoning_decision() + self._check_context_fits( + effective_system_prompt, messages=messages, reasoning=reasoning + ) response = self._generate_anthropic( effective_system_prompt, user_prompt, call_type=call_type, messages=messages, + reasoning=reasoning, ) # On success, accumulate the user message + assistant response in history @@ -1740,12 +1760,16 @@ def _generate_response_with_session_sync( f"sending {len(messages)} msgs to Converse" ) - self._check_context_fits(effective_system_prompt, messages=messages) + reasoning = self.reasoning_decision() + self._check_context_fits( + effective_system_prompt, messages=messages, reasoning=reasoning + ) response = self._generate_bedrock( effective_system_prompt, user_prompt, call_type=call_type, messages=messages, + reasoning=reasoning, ) # On success, accumulate the user message + assistant response in @@ -2044,6 +2068,8 @@ def _generate_openai( call_type: Optional[str] = None, messages_override: Optional[List[Dict[str, Any]]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Delegate to the chat_completions transport (Phase 2).""" return _transports.chat_completions.generate_openai( @@ -2053,15 +2079,7 @@ def _generate_openai( call_type=call_type, messages_override=messages_override, json_mode=json_mode, - ) - - @profile("llm_ollama_call", OperationCategory.LLM) - def _generate_ollama( - self, system_prompt: str | None, user_prompt: str, json_mode: bool = True - ) -> Dict[str, Any]: - """Delegate to the chat_completions transport's Ollama path (Phase 2).""" - return _transports.chat_completions.generate_ollama( - self, system_prompt, user_prompt, json_mode=json_mode + reasoning=reasoning, ) @profile("llm_gemini_call", OperationCategory.LLM) @@ -2072,6 +2090,8 @@ def _generate_gemini( call_type: Optional[str] = None, contents_override: Optional[List[Dict[str, Any]]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Delegate to the gemini_native transport (Phase 2).""" return _transports.gemini_native.generate( @@ -2081,15 +2101,9 @@ def _generate_gemini( call_type=call_type, contents_override=contents_override, json_mode=json_mode, + reasoning=reasoning, ) - @profile("llm_byteplus_call", OperationCategory.LLM) - def _generate_byteplus( - self, system_prompt: str | None, user_prompt: str - ) -> Dict[str, Any]: - """Delegate to the byteplus_responses transport (Phase 2).""" - return _transports.byteplus_responses.generate(self, system_prompt, user_prompt) - def _parse_responses_api_content(self, result: Dict[str, Any]) -> str: """Parse content from BytePlus Responses API response. @@ -2119,6 +2133,8 @@ def _generate_anthropic( user_prompt: str, call_type: Optional[str] = None, messages: Optional[List[dict]] = None, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Delegate to the anthropic_messages transport (Phase 2).""" return _transports.anthropic_messages.generate( @@ -2127,6 +2143,7 @@ def _generate_anthropic( user_prompt, call_type=call_type, messages=messages, + reasoning=reasoning, ) # ─────────── Bedrock model capability detection ─────────────────── @@ -2152,6 +2169,8 @@ def _generate_bedrock( user_prompt: str, call_type: Optional[str] = None, messages: Optional[List[dict]] = None, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Delegate to the bedrock_converse transport (Phase 2).""" return _transports.bedrock_converse.generate( @@ -2160,6 +2179,7 @@ def _generate_bedrock( user_prompt, call_type=call_type, messages=messages, + reasoning=reasoning, ) # ─────────────────── CLI helper for ad‑hoc testing ─────────────────── diff --git a/agent_core/core/impl/llm/reasoning_wire.py b/agent_core/core/impl/llm/reasoning_wire.py index 32d9189e..addbff2b 100644 --- a/agent_core/core/impl/llm/reasoning_wire.py +++ b/agent_core/core/impl/llm/reasoning_wire.py @@ -1,12 +1,18 @@ # -*- coding: utf-8 -*- -"""Render a reasoning decision into request fields, one function per transport. +"""Apply a reasoning decision to one request, one function per transport. The table, the choices, and the level policy live in agent_core/core/models/reasoning.py; this module only knows how each -transport spells a ``ReasoningDecision``. Callers skip these functions -entirely when a model has no rule, so a model without a rule never gains a -field here. A decision that sends nothing (the provider default, or off on a -model whose default is no reasoning) renders as no fields. +transport carries a ``ReasoningDecision``. The interface resolves the +decision once per request and hands the same decision to its context-window +check and to the transport, whose ``for_*`` function below turns it into the +request's output cap, temperature permission, and reasoning fields. + +A request without a decision (the model has no rule) keeps the transport's +own cap and temperature and gains no field, so its payload is exactly the +one sent before per-model rules existed. A decision that sends nothing (the +provider default, or off on a model whose default is no reasoning) renders +as no fields. A decision whose wire the transport cannot express is a bug in the rules table (a row filed under the wrong provider, or an explicit off on a wire @@ -17,13 +23,65 @@ from __future__ import annotations -from typing import Any, Dict, Tuple +from dataclasses import dataclass +from typing import Any, Callable, Dict, Generic, Optional, Tuple, TypeVar from agent_core.core.models.reasoning import ReasoningDecision, ReasoningWire #: The disable value of the OpenAI-style effort parameter. EFFORT_OFF = "none" +FieldsT = TypeVar("FieldsT") + + +@dataclass(frozen=True) +class RequestReasoning(Generic[FieldsT]): + """How one request carries its reasoning decision on one transport.""" + + #: Output-token cap: the transport's own cap, raised while reasoning + #: (reasoning tokens count against it), never lowered. + output_cap: int + #: Whether the request may carry an explicit temperature. + send_temperature: bool + #: The transport's reasoning fields; empty when nothing is sent. + fields: FieldsT + + +def _for_request( + decision: Optional[ReasoningDecision], + base_cap: int, + render: Callable[[ReasoningDecision], FieldsT], + no_fields: FieldsT, +) -> RequestReasoning[FieldsT]: + if decision is None: + return RequestReasoning(base_cap, True, no_fields) + return RequestReasoning( + output_cap=decision.output_cap(base_cap), + send_temperature=not decision.omit_temperature, + fields=render(decision), + ) + + +def for_chat_completions( + decision: Optional[ReasoningDecision], base_cap: int +) -> RequestReasoning[Tuple[Dict[str, Any], Dict[str, Any]]]: + """Chat Completions; fields are (top-level kwargs, extra_body).""" + return _for_request(decision, base_cap, chat_completions_fields, ({}, {})) + + +def for_anthropic( + decision: Optional[ReasoningDecision], base_cap: int +) -> RequestReasoning[Dict[str, Any]]: + """Anthropic Messages, and Claude on Bedrock Converse.""" + return _for_request(decision, base_cap, anthropic_fields, {}) + + +def for_gemini( + decision: Optional[ReasoningDecision], base_cap: int +) -> RequestReasoning[Dict[str, Any]]: + """Gemini generateContent; fields are GeminiClient keyword arguments.""" + return _for_request(decision, base_cap, gemini_thinking_kwargs, {}) + def _unsupported(decision: ReasoningDecision, transport: str) -> ValueError: what = "an explicit off" if decision.off else f"wire {decision.wire.value!r}" diff --git a/agent_core/core/impl/llm/transports/__init__.py b/agent_core/core/impl/llm/transports/__init__.py index 6646e72f..16aff246 100644 --- a/agent_core/core/impl/llm/transports/__init__.py +++ b/agent_core/core/impl/llm/transports/__init__.py @@ -8,8 +8,11 @@ session state stays on LLMInterface (NFR-3 in docs/PROVIDER_LAYER_CATCHUP.md). TRANSPORTS maps ProviderProfile.wire -> the transport's generate callable -with signature (iface, system_prompt, user_prompt, json_mode=True) -> response dict +with signature (iface, system_prompt, user_prompt, json_mode=True, *, +reasoning) -> response dict ({"content", "tokens_used", "cached_tokens"?, "error"?, "error_info_obj"?}). +``reasoning`` is the request's ReasoningDecision, resolved once by the +interface (None when the model has no reasoning rule). Session-mode entry points with richer signatures are exposed as module functions and called by the session dispatcher on LLMInterface. """ diff --git a/agent_core/core/impl/llm/transports/anthropic_messages.py b/agent_core/core/impl/llm/transports/anthropic_messages.py index 56639b0f..abafa6fd 100644 --- a/agent_core/core/impl/llm/transports/anthropic_messages.py +++ b/agent_core/core/impl/llm/transports/anthropic_messages.py @@ -13,6 +13,7 @@ from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.core.models.reasoning import ReasoningDecision from agent_core.utils.logger import logger # Anthropic requires max_tokens; 16384 (Claude 4 default) avoids truncation. @@ -27,6 +28,8 @@ def generate( call_type: Optional[str] = None, messages: Optional[List[dict]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Generate response using Anthropic with prompt caching. @@ -53,6 +56,9 @@ def generate( When provided, uses extended 1-hour TTL for better cache hit rates. messages: Optional pre-built messages list for multi-turn sessions. When provided, used instead of building a single-turn message. + reasoning: This request's reasoning decision, resolved once by the + interface (None: the model has no rule, so the request is + shaped exactly as it was before per-model rules). Cache hits are logged when `cache_read_input_tokens` > 0 in the response. """ @@ -75,20 +81,14 @@ def generate( if not iface._anthropic_client: raise RuntimeError("Anthropic client was not initialised.") - # Reasoning default for this exact model (None: no rule, so the - # request is shaped exactly as it was before reasoning defaults). - reasoning = iface.reasoning_decision() + # Claude rows' reasoning caps stay below the SDK's non-streaming + # ceiling at every level. + applied = reasoning_wire.for_anthropic(reasoning, _DEFAULT_MAX_TOKENS) # Build the message - use pre-built messages for multi-turn, or single-turn. - # Thinking tokens count against max_tokens, so a reasoning default - # raises it (staying below the SDK's non-streaming ceiling). message_kwargs: Dict[str, Any] = { "model": iface.model, - "max_tokens": ( - _DEFAULT_MAX_TOKENS - if reasoning is None - else reasoning.output_cap(_DEFAULT_MAX_TOKENS) - ), + "max_tokens": applied.output_cap, "messages": messages if messages is not None else [ @@ -122,13 +122,12 @@ def generate( # Short prompt - use simple string format (no caching) message_kwargs["system"] = system_prompt - if reasoning is not None: - message_kwargs.update(reasoning_wire.anthropic_fields(reasoning)) + message_kwargs.update(applied.fields) # Thinking is incompatible with temperature on Claude 4.5/4.6, and - # Claude 4.7+ rejects any non-default temperature, so rows with - # reasoning drop it (the model then uses its default). - if reasoning is None or not reasoning.omit_temperature: + # Claude 4.7+ rejects any non-default temperature, so those requests + # drop it (the model then uses its default). + if applied.send_temperature: message_kwargs["extra_body"] = {"temperature": iface.temperature} response = iface._anthropic_client.messages.create(**message_kwargs) diff --git a/agent_core/core/impl/llm/transports/bedrock_converse.py b/agent_core/core/impl/llm/transports/bedrock_converse.py index 7c41d07d..3af9884b 100644 --- a/agent_core/core/impl/llm/transports/bedrock_converse.py +++ b/agent_core/core/impl/llm/transports/bedrock_converse.py @@ -14,6 +14,7 @@ from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.core.models.reasoning import ReasoningDecision from agent_core.utils.logger import logger @@ -25,6 +26,8 @@ def generate( call_type: Optional[str] = None, messages: Optional[List[dict]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Generate response via AWS Bedrock Converse API with prompt caching. @@ -47,6 +50,9 @@ def generate( needed and placing it in messages lets the cache grow with the conversation). When messages is None, falls back to a fresh single-turn call with cachePoint on the system block. + reasoning: This request's reasoning decision, resolved once by the + interface (None: the model has no rule, so the request is shaped + exactly as it was before per-model rules). """ token_count_input = token_count_output = 0 total_tokens = 0 @@ -71,31 +77,22 @@ def generate( else [{"role": "user", "content": [{"text": user_prompt}]}] ) - # Reasoning default for this exact model (None: no rule, so the - # request is shaped exactly as it was before reasoning defaults). - reasoning = iface.reasoning_decision() + # Claude on Converse takes the Messages-API thinking/effort keys + # through additionalModelRequestFields, with the same cap and + # temperature consequences. + applied = reasoning_wire.for_anthropic(reasoning, iface.max_tokens) + inference_config: Dict[str, Any] = {} + if applied.send_temperature: + inference_config["temperature"] = iface.temperature + inference_config["maxTokens"] = applied.output_cap converse_kwargs: Dict[str, Any] = { "modelId": iface.model, "messages": converse_messages, - "inferenceConfig": { - "temperature": iface.temperature, - "maxTokens": iface.max_tokens, - }, + "inferenceConfig": inference_config, } - if reasoning is not None: - # Claude on Converse takes the Messages-API thinking/effort keys - # through additionalModelRequestFields. Thinking tokens count - # against maxTokens, and thinking is incompatible with (4.5/4.6) - # or rejects (4.7+) a non-default temperature. - reasoning_fields = reasoning_wire.anthropic_fields(reasoning) - if reasoning_fields: - converse_kwargs["additionalModelRequestFields"] = reasoning_fields - converse_kwargs["inferenceConfig"]["maxTokens"] = reasoning.output_cap( - iface.max_tokens - ) - if reasoning.omit_temperature: - del converse_kwargs["inferenceConfig"]["temperature"] + if applied.fields: + converse_kwargs["additionalModelRequestFields"] = applied.fields if system_prompt: # When messages already carry a cachePoint (multi-turn first diff --git a/agent_core/core/impl/llm/transports/byteplus_responses.py b/agent_core/core/impl/llm/transports/byteplus_responses.py index 5e726aae..ea5db649 100644 --- a/agent_core/core/impl/llm/transports/byteplus_responses.py +++ b/agent_core/core/impl/llm/transports/byteplus_responses.py @@ -22,12 +22,18 @@ get_cache_metrics, ) from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.core.models.reasoning import ReasoningDecision from agent_core.utils.logger import logger @profile("llm_byteplus_call", OperationCategory.LLM) def generate( - iface, system_prompt: str | None, user_prompt: str, json_mode: bool = True + iface, + system_prompt: str | None, + user_prompt: str, + json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Generate response using BytePlus with automatic prefix caching. @@ -35,6 +41,8 @@ def generate( ``json_mode`` is accepted for transport-signature uniformity but unused: the Responses API has no json_object knob here — JSON is prompt-instructed. + ``reasoning`` is accepted for the same reason: no BytePlus model has a + reasoning rule, so it is always None. """ config = get_cache_config() # Use prefix caching if: diff --git a/agent_core/core/impl/llm/transports/chat_completions.py b/agent_core/core/impl/llm/transports/chat_completions.py index f0871153..c58977e1 100644 --- a/agent_core/core/impl/llm/transports/chat_completions.py +++ b/agent_core/core/impl/llm/transports/chat_completions.py @@ -22,6 +22,7 @@ from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error, provider_display_name +from agent_core.core.models.reasoning import ReasoningDecision from agent_core.core.models.registry import ( get_registry as _get_registry, supports_prompt_cache_key as _supports_pck, @@ -51,6 +52,8 @@ def generate_openai( call_type: Optional[str] = None, messages_override: Optional[List[Dict[str, Any]]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Generate response using OpenAI with automatic prompt caching. @@ -73,6 +76,9 @@ def generate_openai( the accumulating prefix via OR's cache_control field. When set, it's sent verbatim — system_prompt is still passed in for cache- key derivation but the request body uses messages_override. + reasoning: This request's reasoning decision, resolved once by the + interface (None: the model has no rule, so the request is shaped + exactly as it was before per-model rules). Cache hits are logged when cached_tokens > 0 in the response. """ @@ -115,14 +121,10 @@ def generate_openai( "model": iface.model, "messages": messages, } - # Reasoning default for this exact model (None: no rule, so the - # request is shaped exactly as it was before reasoning defaults). - reasoning = iface.reasoning_decision() + applied = reasoning_wire.for_chat_completions(reasoning, iface.max_tokens) _profile = _get_registry().get(iface.provider) _temp = _resolve_temperature(_profile, iface.temperature) - if _temp is not _OMIT_TEMPERATURE and not ( - reasoning is not None and reasoning.omit_temperature - ): + if _temp is not _OMIT_TEMPERATURE and applied.send_temperature: request_kwargs["temperature"] = _temp # Output tokens: cap the VALUE to the provider's output limit (several @@ -130,13 +132,8 @@ def generate_openai( # when it's exceeded), and pick the FIELD NAME per provider policy # (profile.uses_max_completion_tokens: OpenAI/Cerebras/MiniMax/Groq # take 'max_completion_tokens'; everyone else legacy 'max_tokens'). - # Reasoning tokens count against this cap, so a reasoning default - # raises it (never lowers it) to leave room for the answer. - _max_tokens_value = ( - iface.max_tokens - if reasoning is None - else reasoning.output_cap(iface.max_tokens) - ) + # The value starts from the reasoning-raised cap. + _max_tokens_value = applied.output_cap if _profile is not None and _profile.max_output_tokens: _max_tokens_value = min(_max_tokens_value, _profile.max_output_tokens) uses_max_completion_tokens = ( @@ -211,12 +208,9 @@ def generate_openai( f"[OPENROUTER] Anthropic cache_control: {cache_control} (model={iface.model})" ) - if reasoning is not None: - reasoning_top, reasoning_extra = reasoning_wire.chat_completions_fields( - reasoning - ) - request_kwargs.update(reasoning_top) - extra_body.update(reasoning_extra) + reasoning_top, reasoning_extra = applied.fields + request_kwargs.update(reasoning_top) + extra_body.update(reasoning_extra) if extra_body: request_kwargs["extra_body"] = extra_body @@ -338,8 +332,18 @@ def generate_openai( @profile("llm_ollama_call", OperationCategory.LLM) def generate_ollama( - iface, system_prompt: str | None, user_prompt: str, json_mode: bool = True + iface, + system_prompt: str | None, + user_prompt: str, + json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: + """Generate a response through Ollama's native ``/api/generate``. + + ``reasoning`` is accepted for transport-signature uniformity but unused: + no Ollama-served model has a reasoning rule, so it is always None. + """ token_count_input = token_count_output = 0 total_tokens = 0 status = "failed" diff --git a/agent_core/core/impl/llm/transports/gemini_native.py b/agent_core/core/impl/llm/transports/gemini_native.py index 4ddc8305..251c965d 100644 --- a/agent_core/core/impl/llm/transports/gemini_native.py +++ b/agent_core/core/impl/llm/transports/gemini_native.py @@ -13,6 +13,7 @@ from agent_core.core.impl.llm import reasoning_wire from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.core.models.reasoning import ReasoningDecision from agent_core.utils.logger import logger @@ -24,6 +25,8 @@ def generate( call_type: Optional[str] = None, contents_override: Optional[List[Dict[str, Any]]] = None, json_mode: bool = True, + *, + reasoning: Optional[ReasoningDecision], ) -> Dict[str, Any]: """Generate response using Gemini with explicit or implicit caching. @@ -45,6 +48,9 @@ def generate( history so Gemini's implicit caching catches the growing stable prefix automatically (caching covers more tokens with every turn without us needing to manage a named cache object). + reasoning: This request's reasoning decision, resolved once by the + interface (None: the model has no rule, so Gemini's own default + thinking is left untouched). Returns: Dict with tokens_used, content, cached_tokens. @@ -53,24 +59,19 @@ def generate( # Per-call reasoning cap, set by callers that pass thinking_budget (e.g. the # entity-judge pipeline). Rides the shared per-call context so no transport - # signature changes. An explicit per-call budget wins over the model's - # reasoning default; otherwise the default for this exact model applies - # (None: no rule, so Gemini's own default thinking is left untouched). + # signature changes. An explicit per-call budget replaces the request's + # reasoning decision. from agent_core.core.impl.llm.interface import _llm_call_ctx caller_budget = (_llm_call_ctx.get() or {}).get("thinking_budget") - reasoning = iface.reasoning_decision() if caller_budget is None else None - if caller_budget is not None: - thinking: Dict[str, Any] = {"thinking_budget": caller_budget} - elif reasoning is not None: - thinking = reasoning_wire.gemini_thinking_kwargs(reasoning) - else: - thinking = {} - # Thinking tokens count against maxOutputTokens, so a reasoning default - # raises the cap (never lowers it) to leave room for the answer. - max_output_tokens = ( - iface.max_tokens if reasoning is None else reasoning.output_cap(iface.max_tokens) + applied = reasoning_wire.for_gemini( + reasoning if caller_budget is None else None, iface.max_tokens + ) + thinking: Dict[str, Any] = ( + applied.fields if caller_budget is None else {"thinking_budget": caller_budget} ) + temperature = iface.temperature if applied.send_temperature else None + max_output_tokens = applied.output_cap token_count_input = token_count_output = 0 cached_tokens = 0 @@ -98,7 +99,7 @@ def generate( iface.model, contents=contents_override, system_prompt=system_prompt, - temperature=iface.temperature, + temperature=temperature, max_output_tokens=max_output_tokens, json_mode=json_mode, **thinking, @@ -130,7 +131,7 @@ def generate( system_prompt=system_prompt, user_prompt=user_prompt, call_type=call_type, - temperature=iface.temperature, + temperature=temperature, max_tokens=max_output_tokens, **thinking, ) @@ -140,7 +141,7 @@ def generate( iface.model, prompt=user_prompt, system_prompt=system_prompt, - temperature=iface.temperature, + temperature=temperature, max_output_tokens=max_output_tokens, json_mode=json_mode, **thinking, diff --git a/tests/llm/test_reasoning_rules.py b/tests/llm/test_reasoning_rules.py index 78abecc8..ff942398 100644 --- a/tests/llm/test_reasoning_rules.py +++ b/tests/llm/test_reasoning_rules.py @@ -746,7 +746,7 @@ def test_gemini_caller_budget_wins_over_the_session(monkeypatch, bound_session): assert config["maxOutputTokens"] == APP_MAX_TOKENS -def test_context_check_reserves_the_reasoning_cap(monkeypatch, bound_session): +def test_context_check_reserves_the_requests_reasoning_cap(monkeypatch, bound_session): import app.config as app_config monkeypatch.setattr(app_config, "get_context_window", lambda: 40_000) @@ -754,19 +754,71 @@ def test_context_check_reserves_the_reasoning_cap(monkeypatch, bound_session): unruled, _ = build_interface(monkeypatch, "openai", "gpt-4o") unruled.max_tokens = APP_MAX_TOKENS - assert unruled._output_reservation() == APP_MAX_TOKENS - unruled._check_context_fits(None, prompt) + assert unruled._output_reservation(unruled.reasoning_decision()) == APP_MAX_TOKENS + unruled._check_context_fits(None, prompt, reasoning=unruled.reasoning_decision()) ruled, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") ruled.max_tokens = APP_MAX_TOKENS - assert ruled._output_reservation() == 32_000 + high = ruled.reasoning_decision() + assert ruled._output_reservation(high) == 32_000 with pytest.raises(LLMContextOverflowError, match="32000 reserved for output"): - ruled._check_context_fits(None, prompt) + ruled._check_context_fits(None, prompt, reasoning=high) + # The request path refuses it before anything is sent. + with pytest.raises(LLMContextOverflowError): + ruled.generate_response(system_prompt=None, user_prompt=prompt) # A session that turned reasoning off reserves only the caller's cap. with bound_session(C.OFF): - assert ruled._output_reservation() == APP_MAX_TOKENS - ruled._check_context_fits(None, prompt) + off = ruled.reasoning_decision() + assert ruled._output_reservation(off) == APP_MAX_TOKENS + ruled._check_context_fits(None, prompt, reasoning=off) + + +def test_each_request_resolves_its_reasoning_once(monkeypatch): + # The context check and the transport share one decision, so a choice + # changed mid-request cannot make them disagree. + iface, rec = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + iface.max_tokens = APP_MAX_TOKENS + resolutions: List[ReasoningDecision] = [] + resolve = iface.reasoning_decision + + def counting_resolve(): + resolutions.append(resolve()) + return resolutions[-1] + + monkeypatch.setattr(iface, "reasoning_decision", counting_resolve) + iface.generate_response(system_prompt=GOLDEN_SYSTEM_PROMPT, user_prompt="hi") + assert len(resolutions) == 1 + + iface.create_session_cache("task", "action_selection", GOLDEN_SYSTEM_PROMPT) + iface.generate_response_with_session( + "task", "action_selection", "hi", log_response=False + ) + assert len(resolutions) == 2 + assert [call["payload"]["reasoning_effort"] for call in rec.calls] == [ + "high", + "high", + ] + + +def test_each_model_and_choice_is_logged_once(monkeypatch, bound_session): + import agent_core.core.impl.llm.interface as interface_module + + iface, _ = build_interface(monkeypatch, "openai", "gpt-5.2-2025-12-11") + with bound_session(C.HIGH, session_id="s-high"): + pass + with bound_session(C.LOW, session_id="s-low"): + pass + logged: List[str] = [] + monkeypatch.setattr(interface_module, "logger", SimpleNamespace(info=logged.append)) + # Two sessions alternating, as concurrent sessions do. + for _ in range(3): + with StateSession.bind("s-high"): + iface.reasoning_decision() + with StateSession.bind("s-low"): + iface.reasoning_decision() + assert len(logged) == 2 + assert "choice=high" in logged[0] and "choice=low" in logged[1] def test_fold_check_reserves_the_same_cap_as_the_pre_send_check( diff --git a/tests/test_bedrock_token_normalization.py b/tests/test_bedrock_token_normalization.py index 58ce2a9d..2aa18a17 100644 --- a/tests/test_bedrock_token_normalization.py +++ b/tests/test_bedrock_token_normalization.py @@ -52,11 +52,8 @@ def test_llm_bedrock_input_includes_cache_and_cached_is_reads_only(): iface = LLMInterface.__new__(LLMInterface) reported = _stub_common(iface) iface._call_log_to_db = lambda *a, **kw: None - # Read by the per-model reasoning lookup every transport performs. - iface._auth_mode = "api_key" - iface._reasoning_logged_for = None - result = iface._generate_bedrock(None, "hi") + result = iface._generate_bedrock(None, "hi", reasoning=None) assert "error" not in result # input = 100 + 900 + 30, the full prompt; cached = reads only diff --git a/tests/test_context_budget.py b/tests/test_context_budget.py index a8ebf7a6..5c3ad86f 100644 --- a/tests/test_context_budget.py +++ b/tests/test_context_budget.py @@ -334,7 +334,9 @@ def test_preflight_refuses_a_request_that_does_not_fit(): iface = _make() with patch.object(app_config, "get_settings", return_value=_settings()): with pytest.raises(LLMContextOverflowError): - iface._check_context_fits("system", "word " * 200_000) + iface._check_context_fits( + "system", "word " * 200_000, reasoning=iface.reasoning_decision() + ) assert iface._consecutive_failures == 0 From c553f08ff7a3640eebfc65540588c26852199ab1 Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 14:17:09 +0900 Subject: [PATCH 11/55] fix incoming message encoding issue --- .../providers/gmail/client.py | 16 +++++-- .../test_listener_text_fidelity.py | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/craftos_integrations/providers/gmail/client.py b/craftos_integrations/providers/gmail/client.py index 30725b77..ece31715 100644 --- a/craftos_integrations/providers/gmail/client.py +++ b/craftos_integrations/providers/gmail/client.py @@ -116,12 +116,16 @@ def _part_charset(part: Dict[str, Any]) -> str: def _decode_part(part: Dict[str, Any]) -> str: - """Decode a part's base64url body using its declared charset. + """Decode a part's base64url body. + + Gmail hands text bodies back already transcoded to UTF-8 but keeps the + original Content-Type charset (a Windows-1252 mail's ``=B7`` arrives as + ``\\xc2\\xb7``), so decoding by the declared charset turns ``·`` into + ``·``. Bytes that are not valid UTF-8 were passed through untranscoded + and are decoded by the declared charset instead. Never raises: an unknown charset or undecodable byte degrades to - replacement characters instead of failing the whole read (a strict - utf-8 decode used to turn any iso-8859-1 / windows-1252 email into an - error that also lost its headers). + replacement characters instead of failing the whole read. """ data = (part.get("body") or {}).get("data", "") if not data: @@ -130,6 +134,10 @@ def _decode_part(part: Dict[str, Any]) -> str: raw = base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) except Exception: return "" + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + pass try: return raw.decode(_part_charset(part), errors="replace") except LookupError: diff --git a/tests/integrations/test_listener_text_fidelity.py b/tests/integrations/test_listener_text_fidelity.py index 99e735e5..f2ed0fc7 100644 --- a/tests/integrations/test_listener_text_fidelity.py +++ b/tests/integrations/test_listener_text_fidelity.py @@ -131,6 +131,49 @@ def test_gmail_listener_converts_html_only_mail(monkeypatch): assert msg.text == "Subject: Top 10 users\nTom & Jerry" +_CP1252_PLAIN = [ + {"name": "Content-Type", "value": 'text/plain; charset="Windows-1252"'} +] + + +def test_gmail_listener_decodes_transcoded_body_as_utf8(monkeypatch): + # Gmail returns the body as UTF-8 but keeps the sender's Windows-1252 + # header; decoding by the header turned "·" into "·". + data = base64.urlsafe_b64encode("01 · Map".encode("utf-8")).decode() + msg = _gmail_client( + monkeypatch, + { + "headers": _GMAIL_HEADERS, + "parts": [ + { + "mimeType": "text/plain", + "headers": _CP1252_PLAIN, + "body": {"data": data}, + } + ], + }, + ) + assert msg.text == "Subject: Top 10 users\n01 · Map" + + +def test_gmail_listener_decodes_untranscoded_body_by_declared_charset(monkeypatch): + data = base64.urlsafe_b64encode("01 · Map".encode("cp1252")).decode() + msg = _gmail_client( + monkeypatch, + { + "headers": _GMAIL_HEADERS, + "parts": [ + { + "mimeType": "text/plain", + "headers": _CP1252_PLAIN, + "body": {"data": data}, + } + ], + }, + ) + assert msg.text == "Subject: Top 10 users\n01 · Map" + + def test_gmail_listener_caps_long_body_and_flags_it(monkeypatch): long_body = "word " * 1000 # 5000 chars msg = _gmail_client( From 991fe15f5464207c5bf96ed97306e2307cf0dc2a Mon Sep 17 00:00:00 2001 From: CraftBot Date: Fri, 2 Oct 2026 23:00:32 +0900 Subject: [PATCH 12/55] very minor UI update --- .../activity/ActivityBlocks.module.css | 6 +- .../src/pages/Chat/ChatPage.module.css | 5 +- .../src/pages/Settings/MemorySettings.tsx | 22 ++++- .../pages/Settings/SettingsPage.module.css | 97 +++++++++++++++---- 4 files changed, 106 insertions(+), 24 deletions(-) diff --git a/app/ui_layer/browser/frontend/src/components/activity/ActivityBlocks.module.css b/app/ui_layer/browser/frontend/src/components/activity/ActivityBlocks.module.css index 4a7cc55f..96eb9229 100644 --- a/app/ui_layer/browser/frontend/src/components/activity/ActivityBlocks.module.css +++ b/app/ui_layer/browser/frontend/src/components/activity/ActivityBlocks.module.css @@ -38,7 +38,9 @@ justify-content: center; } -/* Icon wrapper — solid background so the icon reads cleanly */ +/* Icon wrapper. Transparent: these rows render on several surfaces + (white chat, the Agent App's off-white panel, dashboard cards), so any + fixed fill would show as a box on all but one of them. */ .gutterIcon { position: relative; z-index: 1; @@ -47,7 +49,6 @@ justify-content: center; width: 22px; height: 22px; - background: var(--bg-primary); } /* Hollow circle for reasoning items */ @@ -55,7 +56,6 @@ width: 10px; height: 10px; border-radius: 50%; - background: var(--bg-primary); border: 1.5px solid var(--border-secondary); flex-shrink: 0; } diff --git a/app/ui_layer/browser/frontend/src/pages/Chat/ChatPage.module.css b/app/ui_layer/browser/frontend/src/pages/Chat/ChatPage.module.css index 7d251417..b767736f 100644 --- a/app/ui_layer/browser/frontend/src/pages/Chat/ChatPage.module.css +++ b/app/ui_layer/browser/frontend/src/pages/Chat/ChatPage.module.css @@ -75,6 +75,7 @@ margin: 0 auto; align-items: center; text-align: left; + min-width: min(360px, 100%); } .messageWrapper.errorWrapper { @@ -163,6 +164,9 @@ .message.system { background: var(--bg-secondary); + /* Light theme's --bg-secondary is near-white, so the border is what + separates the bubble from the panel. */ + border: 1px solid var(--border-primary); color: var(--text-secondary); font-size: var(--text-sm); } @@ -203,7 +207,6 @@ .detailsBody { margin-top: var(--space-1); padding: var(--space-2); - border-left: 2px solid var(--border-primary); background: var(--bg-tertiary); border-radius: var(--radius-sm); font-size: var(--text-sm); diff --git a/app/ui_layer/browser/frontend/src/pages/Settings/MemorySettings.tsx b/app/ui_layer/browser/frontend/src/pages/Settings/MemorySettings.tsx index b2507fb3..01046e5b 100644 --- a/app/ui_layer/browser/frontend/src/pages/Settings/MemorySettings.tsx +++ b/app/ui_layer/browser/frontend/src/pages/Settings/MemorySettings.tsx @@ -63,7 +63,11 @@ export function MemorySettings() { const saveTimerRef = useRef(undefined) // Custom threshold slider (drag the picker along the track). const gateBarRef = useRef(null) + const gateThumbRef = useRef(null) const gateDraggingRef = useRef(false) + // Pointer's distance from the marker's center when a drag starts on the + // marker itself, so grabbing it off-center doesn't make it jump. + const gateGrabOffsetRef = useRef(0) // Confirm modal const { modalProps: confirmModalProps, confirm } = useConfirmModal() @@ -127,13 +131,21 @@ export function MemorySettings() { const el = gateBarRef.current if (!el) return const rect = el.getBoundingClientRect() - const pct = rect.width > 0 ? (clientX - rect.left) / rect.width : 0 + const x = clientX - gateGrabOffsetRef.current + const pct = rect.width > 0 ? (x - rect.left) / rect.width : 0 setThreshold(Math.round(Math.min(1, Math.max(0, pct)) * thresholdMax)) } const onGatePointerDown = (e: RPointerEvent) => { if (!memoryEnabled) return gateDraggingRef.current = true e.currentTarget.setPointerCapture(e.pointerId) + const thumb = gateThumbRef.current + if (thumb && thumb.contains(e.target as Node)) { + const r = thumb.getBoundingClientRect() + gateGrabOffsetRef.current = e.clientX - (r.left + r.width / 2) + } else { + gateGrabOffsetRef.current = 0 + } setThresholdFromPointer(e.clientX) } const onGatePointerMove = (e: RPointerEvent) => { @@ -290,9 +302,15 @@ export function MemorySettings() { style={{ width: `${Math.min(100, (unprocessed / thresholdMax) * 100)}%` }} />
+ > + +
+
diff --git a/app/ui_layer/browser/frontend/src/pages/Settings/SettingsPage.module.css b/app/ui_layer/browser/frontend/src/pages/Settings/SettingsPage.module.css index e43ea317..0ff07a84 100644 --- a/app/ui_layer/browser/frontend/src/pages/Settings/SettingsPage.module.css +++ b/app/ui_layer/browser/frontend/src/pages/Settings/SettingsPage.module.css @@ -3035,11 +3035,17 @@ width: 100%; box-sizing: border-box; height: 8px; - margin: 10px 0 4px; + /* Top room for the threshold marker's tag, which sits above the bar. */ + margin: 30px 0 10px; border-radius: 999px; background: var(--bg-tertiary); cursor: pointer; touch-action: none; + user-select: none; +} + +.gateTrack:not(.gateDisabled):active { + cursor: grabbing; } .gateTrack:focus-visible { @@ -3063,22 +3069,54 @@ transition: width 0.3s ease; } +/* Threshold marker: a red tag above the bar pointing down into a red dashed + line that crosses it. The whole 16px-wide column is the grab target, so + both the tag and the line drag (pointer-downs on it bubble to the track's + handlers). The halo keeps the red readable over the orange fill. */ .gateThumb { position: absolute; - top: 50%; - width: 18px; - height: 18px; - border-radius: 50%; - background: var(--text-primary); - border: 3px solid var(--color-primary); - transform: translate(-50%, -50%); - box-shadow: 0 1px 4px rgba(0, 0, 0, 0.4); - pointer-events: none; + top: -26px; + bottom: -6px; + width: 16px; + transform: translateX(-50%); + display: flex; + flex-direction: column; + align-items: center; + color: var(--color-error); + filter: drop-shadow(0 0 1px var(--bg-primary)); + cursor: grab; +} + +.gateDisabled .gateThumb { + cursor: default; +} + +.gateTag { + flex-shrink: 0; + fill: currentColor; + transform-origin: 50% 100%; + transition: transform var(--transition-fast); +} + +.gateTrack:not(.gateDisabled) .gateThumb:hover .gateTag, +.gateTrack:not(.gateDisabled):active .gateTag { + transform: scale(1.15); +} + +/* Dashes drawn with a gradient so their length is exact at this height. */ +.gateLine { + flex: 1; + width: 2px; + background: repeating-linear-gradient( + to bottom, + currentColor 0 3px, + transparent 3px 5px + ); } /* Legend under the track: names the two things on the bar (orange fill = - events waiting, ringed dot = the draggable minimum) so the numbers need - no decoding. */ + events waiting, red tag = the draggable minimum) so the numbers need no + decoding. */ .gateLegend { display: flex; align-items: center; @@ -3104,16 +3142,39 @@ flex-shrink: 0; } +/* Miniature of the threshold marker: red tag over a dashed stem. */ .gateSwatchThumb { - width: 10px; - height: 10px; - border-radius: 50%; - background: var(--text-primary); - border: 2px solid var(--color-primary); - box-sizing: border-box; + position: relative; + width: 8px; + height: 13px; flex-shrink: 0; } +.gateSwatchThumb::before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 8px; + height: 8px; + background: var(--color-error); + clip-path: polygon(0 0, 100% 0, 100% 60%, 50% 100%, 0 60%); +} + +.gateSwatchThumb::after { + content: ''; + position: absolute; + top: 8px; + bottom: 0; + left: 3px; + width: 2px; + background: repeating-linear-gradient( + to bottom, + var(--color-error) 0 2px, + transparent 2px 3px + ); +} + /* One-line verdict on the next scheduled run: green when enough events have accumulated (run will fire), muted while it would still be skipped. */ .gateStatus { From c86c275a583743794f8530b14a51ea55c726eb62 Mon Sep 17 00:00:00 2001 From: zfoong Date: Mon, 5 Oct 2026 15:07:08 +0900 Subject: [PATCH 13/55] make gmail able to send inline email --- .../providers/gmail/GUIDANCE.md | 4 + .../providers/gmail/client.py | 297 ++++++++++++------ .../providers/gmail/operations.py | 44 ++- tests/integrations/test_google_providers.py | 172 ++++++++++ 4 files changed, 414 insertions(+), 103 deletions(-) diff --git a/craftos_integrations/providers/gmail/GUIDANCE.md b/craftos_integrations/providers/gmail/GUIDANCE.md index d324d43a..e1da4256 100644 --- a/craftos_integrations/providers/gmail/GUIDANCE.md +++ b/craftos_integrations/providers/gmail/GUIDANCE.md @@ -19,6 +19,10 @@ Email — read, search, send, drafts, labels, threads. empty, don't answer a flat "no updates" — say there's nothing unread and either offer or show the most recent messages (`unread_only=false`). - `send_gmail` with no `to` sends to the connected account's own address. +- To show an image inside the email (not as an attachment), pass its local + path in `inline_images` and reference it from an HTML body (`html=true`) + as `` (cid = the file name). Use `attachments` + only when the user wants a downloadable file. - Prefer `trash_gmail` (reversible) over `delete_gmail` (permanent). - Use Gmail search syntax in `search_gmail` (`from:`, `subject:`, `newer_than:7d`, `has:attachment`, ...). diff --git a/craftos_integrations/providers/gmail/client.py b/craftos_integrations/providers/gmail/client.py index ece31715..2780f2ab 100644 --- a/craftos_integrations/providers/gmail/client.py +++ b/craftos_integrations/providers/gmail/client.py @@ -21,10 +21,13 @@ import mimetypes import os import re +import urllib.parse +import urllib.request from dataclasses import dataclass from datetime import timezone from email import encoders from email.mime.base import MIMEBase +from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from html.parser import HTMLParser @@ -266,6 +269,163 @@ def _readable_body(texts: Dict[str, str]) -> Tuple[str, str]: return "", "none" +# ----- Outgoing MIME ----- + +# — group 3 is the src value. +_IMG_SRC = re.compile(r"""(]*?\bsrc\s*=\s*)(["'])(.*?)\2""", re.I | re.S) +_CID_UNSAFE = re.compile(r"[^A-Za-z0-9._-]") + + +def _attach_files(msg: MIMEMultipart, paths: Optional[List[str]]) -> None: + """Attach local files as regular attachments (missing paths are skipped).""" + for file_path in paths or []: + if not os.path.isfile(file_path): + continue + mime_type, _ = mimetypes.guess_type(file_path) + if mime_type is None: + mime_type = "application/octet-stream" + maintype, subtype = mime_type.split("/", 1) + with open(file_path, "rb") as f: + part = MIMEBase(maintype, subtype) + part.set_payload(f.read()) + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f'attachment; filename="{os.path.basename(file_path)}"', + ) + msg.attach(part) + + +def _src_to_path(src: str) -> str: + """An ```` value as a local path (``file://`` URLs decoded).""" + if src.lower().startswith("file:"): + return urllib.request.url2pathname(urllib.parse.urlparse(src).path) + return src + + +def _build_message( + body: str, + *, + as_html: bool = False, + inline_images: Optional[List[str]] = None, + attachments: Optional[List[str]] = None, +) -> MIMEMultipart: + """The MIME tree every outgoing mail (send / reply / draft) is built on. + + A plain-text body with no images stays a single text/plain part. HTML, + or any inline image, gives:: + + mixed + ├── related (only with inline images) + │ ├── alternative + │ │ ├── text/plain (derived from the HTML) + │ │ └── text/html + │ └── image/* parts (Content-ID, disposition inline) + └── attachments + + Inline images come from ``inline_images`` and from any HTML ```` + naming a local file. Each is served as ``cid:``: an + ```` holding the image's path or file name is rewritten to its + cid, and an image the HTML never references is appended below the body + so it still renders inline. A missing or non-image file raises + ValueError — not sending beats sending a mail with a broken picture. + """ + if isinstance(inline_images, str): # a lone path instead of a list + inline_images = [inline_images] + msg = MIMEMultipart() + if not as_html and not inline_images: + msg.attach(MIMEText(body, "plain")) + _attach_files(msg, attachments) + return msg + + html_body = ( + body + if as_html + else "
" + html.escape(body).replace("\n", "
\n") + "
" + ) + + # normalized path -> (path, cid), in embed order; aliases maps the names + # a "cid:" reference may use (cid, raw file name) to the cid. + images: Dict[str, Tuple[str, str]] = {} + aliases: Dict[str, str] = {} + + def _embed(path: str) -> str: + key = os.path.normcase(os.path.abspath(path)) + if key in images: + return images[key][1] + if not os.path.isfile(path): + raise ValueError(f"Inline image not found: {path}") + mime_type, _ = mimetypes.guess_type(path) + if not (mime_type or "").startswith("image/"): + raise ValueError(f"Inline image is not an image file: {path}") + name = os.path.basename(path) + stem, ext = os.path.splitext(_CID_UNSAFE.sub("_", name)) + cid, n = stem + ext, 2 + while cid in aliases.values(): + cid, n = f"{stem}-{n}{ext}", n + 1 + images[key] = (path, cid) + aliases.setdefault(cid, cid) + aliases.setdefault(name, cid) + return cid + + for path in inline_images or []: + _embed(path) + + referenced: set = set() + + def _rewrite(m: re.Match) -> str: + src = html.unescape(m.group(3)).strip() + if src.lower().startswith("cid:"): + cid = aliases.get(urllib.parse.unquote(src[4:])) + else: + path = _src_to_path(src) + cid = _embed(path) if os.path.isfile(path) else None + if cid is None: + return m.group(0) + referenced.add(cid) + return f"{m.group(1)}{m.group(2)}cid:{cid}{m.group(2)}" + + html_body = _IMG_SRC.sub(_rewrite, html_body) + + unplaced = "".join( + f'
{cid}
' + for _, cid in images.values() + if cid not in referenced + ) + if unplaced: + close = html_body.lower().rfind("") + html_body = ( + html_body[:close] + unplaced + html_body[close:] + if close >= 0 + else html_body + unplaced + ) + + # utf-8 → base64 bodies, so long HTML lines never break the 998-char limit. + alternative = MIMEMultipart("alternative") + plain = _html_to_text(html_body) if as_html else body + alternative.attach(MIMEText(plain, "plain", "utf-8")) + alternative.attach(MIMEText(html_body, "html", "utf-8")) + + if images: + related = MIMEMultipart("related") + related.attach(alternative) + for path, cid in images.values(): + subtype = mimetypes.guess_type(path)[0].split("/", 1)[1] + with open(path, "rb") as f: + part = MIMEImage(f.read(), _subtype=subtype) + part.add_header("Content-ID", f"<{cid}>") + part.add_header( + "Content-Disposition", "inline", filename=os.path.basename(path) + ) + related.attach(part) + msg.attach(related) + else: + msg.attach(alternative) + + _attach_files(msg, attachments) + return msg + + GMAIL = IntegrationSpec( name="gmail", cred_class=GoogleCredential, @@ -541,31 +701,15 @@ def _encode_email( subject: str, body: str, attachments: Optional[List[str]] = None, + html: bool = False, + inline_images: Optional[List[str]] = None, ) -> str: - msg = MIMEMultipart() + msg = _build_message( + body, as_html=html, inline_images=inline_images, attachments=attachments + ) msg["to"] = to_email msg["from"] = from_email msg["subject"] = subject - msg.attach(MIMEText(body, "plain")) - - if attachments: - for file_path in attachments: - if not os.path.isfile(file_path): - continue - mime_type, _ = mimetypes.guess_type(file_path) - if mime_type is None: - mime_type = "application/octet-stream" - maintype, subtype = mime_type.split("/", 1) - with open(file_path, "rb") as f: - part = MIMEBase(maintype, subtype) - part.set_payload(f.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f'attachment; filename="{os.path.basename(file_path)}"', - ) - msg.attach(part) - return base64.urlsafe_b64encode(msg.as_bytes()).decode() def send_email( @@ -575,6 +719,8 @@ def send_email( body: str = "", from_email: Optional[str] = None, attachments: Optional[List[str]] = None, + html: bool = False, + inline_images: Optional[List[str]] = None, ) -> Result: cred = self._load() sender = from_email or cred.email @@ -582,7 +728,12 @@ def send_email( # Agent App's daily digest, an agent self-notification) should never # need to know or store the user's address — identity is CraftBot's. recipient = to or cred.email - raw = self._encode_email(recipient, sender, subject, body, attachments) + try: + raw = self._encode_email( + recipient, sender, subject, body, attachments, html, inline_images + ) + except ValueError as e: + return {"error": str(e)} return http_request( "POST", f"{GMAIL_API_BASE}/users/me/messages/send", @@ -802,7 +953,15 @@ def reply_to_message( body: str, reply_all: bool = False, attachments: Optional[List[str]] = None, + html: bool = False, + inline_images: Optional[List[str]] = None, ) -> Result: + try: + msg = _build_message( + body, as_html=html, inline_images=inline_images, attachments=attachments + ) + except ValueError as e: + return {"error": str(e)} info = self._fetch_reply_headers(message_id) if info.get("_error"): return {"error": info["_error"]} @@ -828,7 +987,6 @@ def reply_to_message( self_email = (cred.email or "").lower() cc_addrs = [a for a in cc_addrs if a and self_email not in a.lower()] - msg = MIMEMultipart() msg["to"] = from_addr msg["from"] = cred.email msg["subject"] = reply_subject @@ -839,25 +997,6 @@ def reply_to_message( msg["References"] = ( (references + " " + msg_id_hdr).strip() if references else msg_id_hdr ) - msg.attach(MIMEText(body, "plain")) - - if attachments: - for file_path in attachments: - if not os.path.isfile(file_path): - continue - mime_type, _ = mimetypes.guess_type(file_path) - if mime_type is None: - mime_type = "application/octet-stream" - maintype, subtype = mime_type.split("/", 1) - with open(file_path, "rb") as f: - part = MIMEBase(maintype, subtype) - part.set_payload(f.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f'attachment; filename="{os.path.basename(file_path)}"', - ) - msg.attach(part) raw = base64.urlsafe_b64encode(msg.as_bytes()).decode() payload: Dict[str, Any] = {"raw": raw} @@ -1026,23 +1165,7 @@ def forward_message( ) msg.attach(part) - if attachments: - for file_path in attachments: - if not os.path.isfile(file_path): - continue - mime_type, _ = mimetypes.guess_type(file_path) - if mime_type is None: - mime_type = "application/octet-stream" - maintype, subtype = mime_type.split("/", 1) - with open(file_path, "rb") as f: - part = MIMEBase(maintype, subtype) - part.set_payload(f.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f'attachment; filename="{os.path.basename(file_path)}"', - ) - msg.attach(part) + _attach_files(msg, attachments) raw = base64.urlsafe_b64encode(msg.as_bytes()).decode() payload: Dict[str, Any] = {"raw": raw} @@ -1179,9 +1302,16 @@ def create_draft( cc: Optional[str] = None, bcc: Optional[str] = None, attachments: Optional[List[str]] = None, + html: bool = False, + inline_images: Optional[List[str]] = None, ) -> Result: cred = self._load() - msg = MIMEMultipart() + try: + msg = _build_message( + body, as_html=html, inline_images=inline_images, attachments=attachments + ) + except ValueError as e: + return {"error": str(e)} msg["to"] = to msg["from"] = cred.email msg["subject"] = subject @@ -1189,25 +1319,6 @@ def create_draft( msg["cc"] = cc if bcc: msg["bcc"] = bcc - msg.attach(MIMEText(body, "plain")) - - if attachments: - for file_path in attachments: - if not os.path.isfile(file_path): - continue - mime_type, _ = mimetypes.guess_type(file_path) - if mime_type is None: - mime_type = "application/octet-stream" - maintype, subtype = mime_type.split("/", 1) - with open(file_path, "rb") as f: - part = MIMEBase(maintype, subtype) - part.set_payload(f.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f'attachment; filename="{os.path.basename(file_path)}"', - ) - msg.attach(part) raw = base64.urlsafe_b64encode(msg.as_bytes()).decode() return http_request( @@ -1231,10 +1342,17 @@ def update_draft( cc: Optional[str] = None, bcc: Optional[str] = None, attachments: Optional[List[str]] = None, + html: bool = False, + inline_images: Optional[List[str]] = None, ) -> Result: """Replaces the draft content (PUT).""" cred = self._load() - msg = MIMEMultipart() + try: + msg = _build_message( + body, as_html=html, inline_images=inline_images, attachments=attachments + ) + except ValueError as e: + return {"error": str(e)} msg["to"] = to msg["from"] = cred.email msg["subject"] = subject @@ -1242,25 +1360,6 @@ def update_draft( msg["cc"] = cc if bcc: msg["bcc"] = bcc - msg.attach(MIMEText(body, "plain")) - - if attachments: - for file_path in attachments: - if not os.path.isfile(file_path): - continue - mime_type, _ = mimetypes.guess_type(file_path) - if mime_type is None: - mime_type = "application/octet-stream" - maintype, subtype = mime_type.split("/", 1) - with open(file_path, "rb") as f: - part = MIMEBase(maintype, subtype) - part.set_payload(f.read()) - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f'attachment; filename="{os.path.basename(file_path)}"', - ) - msg.attach(part) raw = base64.urlsafe_b64encode(msg.as_bytes()).decode() return http_request( diff --git a/craftos_integrations/providers/gmail/operations.py b/craftos_integrations/providers/gmail/operations.py index 5777df02..46aae89e 100644 --- a/craftos_integrations/providers/gmail/operations.py +++ b/craftos_integrations/providers/gmail/operations.py @@ -22,6 +22,26 @@ from .client import clean_snippet +# Body-format inputs shared by every op that composes a message +# (send / reply / create + update draft). +_HTML_INPUT = { + "type": "boolean", + "description": "Treat body as HTML instead of plain text (default false).", + "example": False, +} +_INLINE_IMAGES_INPUT = { + "type": "array", + "description": ( + "Optional local image file paths shown INSIDE the email body (not as " + "attachments). Place each in an HTML body with \">, e.g. ; an holding a " + "local path is embedded too. Images not placed in the body are shown " + "below it, so a plain-text body works as well." + ), + "example": [], +} + + def _get_gmail_thread_op() -> Operation: """get_gmail_thread with the lean-shaping of the raw thread.""" base = client_op( @@ -180,9 +200,11 @@ def build_operations() -> List[Operation]: }, "body": { "type": "string", - "description": "Email body text.", + "description": "Email body (plain text, or HTML with html=true).", "example": "Hi, here are the notes...", }, + "html": _HTML_INPUT, + "inline_images": _INLINE_IMAGES_INPUT, "attachments": { "type": "array", "description": "Optional list of file paths to attach.", @@ -195,6 +217,8 @@ def build_operations() -> List[Operation]: "subject": d["subject"], "body": d["body"], "attachments": d.get("attachments"), + "html": bool(d.get("html", False)), + "inline_images": d.get("inline_images"), }, ), client_op( @@ -316,7 +340,7 @@ def build_operations() -> List[Operation]: }, "body": { "type": "string", - "description": "Reply body text.", + "description": "Reply body (plain text, or HTML with html=true).", "example": "Thanks — confirmed for Tuesday.", }, "reply_all": { @@ -324,6 +348,8 @@ def build_operations() -> List[Operation]: "description": "Reply to all recipients.", "example": False, }, + "html": _HTML_INPUT, + "inline_images": _INLINE_IMAGES_INPUT, }, ), client_op( @@ -641,9 +667,11 @@ def build_operations() -> List[Operation]: }, "body": { "type": "string", - "description": "Draft body text.", + "description": "Draft body (plain text, or HTML with html=true).", "example": "Draft text...", }, + "html": _HTML_INPUT, + "inline_images": _INLINE_IMAGES_INPUT, }, ), client_op( @@ -662,7 +690,13 @@ def build_operations() -> List[Operation]: }, "to": {"type": "string", "description": "Recipient.", "example": ""}, "subject": {"type": "string", "description": "Subject.", "example": ""}, - "body": {"type": "string", "description": "Body text.", "example": ""}, + "body": { + "type": "string", + "description": "Body (plain text, or HTML with html=true).", + "example": "", + }, + "html": _HTML_INPUT, + "inline_images": _INLINE_IMAGES_INPUT, "cc": {"type": "string", "description": "Optional CC.", "example": ""}, "bcc": { "type": "string", @@ -683,6 +717,8 @@ def build_operations() -> List[Operation]: "cc": d.get("cc") or None, "bcc": d.get("bcc") or None, "attachments": d.get("attachments"), + "html": bool(d.get("html", False)), + "inline_images": d.get("inline_images"), }, ), client_op( diff --git a/tests/integrations/test_google_providers.py b/tests/integrations/test_google_providers.py index 57d717ca..d175eeb4 100644 --- a/tests/integrations/test_google_providers.py +++ b/tests/integrations/test_google_providers.py @@ -8,10 +8,13 @@ from __future__ import annotations import asyncio +import base64 +import email import pytest import craftos_integrations.providers._google as google_mod +import craftos_integrations.providers.gmail.client as gmail_client_mod from craftos_integrations.core.storage import FileCredentialStore from craftos_integrations.core.system import IntegrationSystem from craftos_integrations.providers.gmail import GmailProvider @@ -133,6 +136,175 @@ def test_operation_error_shape_is_agent_friendly(system, monkeypatch): assert "403" in result["message"] +# ----- Outgoing MIME: HTML + inline images ----- + +PNG = b"\x89PNG\r\n\x1a\nfake-png-bytes" + + +@pytest.fixture +def sent(monkeypatch): + """Captures the parsed MIME message of every Gmail send/draft call.""" + captured = [] + + def fake_http(method, url, **kwargs): + payload = kwargs["json"] + raw = (payload.get("message") or payload)["raw"] + captured.append(email.message_from_bytes(base64.urlsafe_b64decode(raw))) + return {"ok": True, "result": {"id": "m1", "threadId": "t1"}} + + monkeypatch.setattr(gmail_client_mod, "http_request", fake_http) + return captured + + +def _parts(msg): + return {p.get_content_type(): p for p in msg.walk()} + + +def _image(tmp_path, name="chart.png"): + path = tmp_path / name + path.write_bytes(PNG) + return str(path) + + +def test_plain_send_is_unchanged(system, sent): + run(system.execute("gmail", "send_gmail", {"subject": "s", "body": "hi"})) + (msg,) = sent + assert [p.get_content_type() for p in msg.get_payload()] == ["text/plain"] + assert msg.get_payload()[0].get_payload(decode=True) == b"hi" + + +def test_html_body_with_cid_inline_image(system, sent, tmp_path): + path = _image(tmp_path) + result = run( + system.execute( + "gmail", + "send_gmail", + { + "subject": "s", + "body": '

Report

', + "html": True, + "inline_images": [path], + }, + ) + ) + assert result["status"] == "success" + (msg,) = sent + parts = _parts(msg) + assert "multipart/related" in parts and "multipart/alternative" in parts + html = parts["text/html"].get_payload(decode=True).decode() + assert html == '

Report

' + assert parts["text/plain"].get_payload(decode=True).decode() == "Report" + img = parts["image/png"] + assert img["Content-ID"] == "" + assert img.get_content_disposition() == "inline" + assert img.get_filename() == "chart.png" + assert img.get_payload(decode=True) == PNG + + +def test_local_img_src_is_embedded_and_rewritten(system, sent, tmp_path): + path = _image(tmp_path, "my chart.png") + run( + system.execute( + "gmail", + "send_gmail", + {"subject": "s", "body": f'', "html": True}, + ) + ) + parts = _parts(sent[0]) + html = parts["text/html"].get_payload(decode=True).decode() + assert html == '' + assert parts["image/png"]["Content-ID"] == "" + + +def test_unplaced_image_is_appended_to_plain_text_body(system, sent, tmp_path): + path = _image(tmp_path) + run( + system.execute( + "gmail", + "send_gmail", + {"subject": "s", "body": "a < b\nsee below", "inline_images": [path]}, + ) + ) + parts = _parts(sent[0]) + html = parts["text/html"].get_payload(decode=True).decode() + assert html.startswith("
a < b
\nsee below
") + assert html.endswith('chart.png
') + assert parts["text/plain"].get_payload(decode=True).decode() == "a < b\nsee below" + + +def test_inline_images_and_attachments_together(system, sent, tmp_path): + img = _image(tmp_path) + doc = tmp_path / "notes.txt" + doc.write_text("notes") + run( + system.execute( + "gmail", + "send_gmail", + { + "subject": "s", + "body": '', + "html": True, + "inline_images": [img], + "attachments": [str(doc)], + }, + ) + ) + (msg,) = sent + top = [p.get_content_type() for p in msg.get_payload()] + assert top == ["multipart/related", "text/plain"] + assert msg.get_payload()[1].get_content_disposition() == "attachment" + + +def test_duplicate_file_names_get_distinct_cids(system, sent, tmp_path): + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + first, second = _image(tmp_path / "a"), _image(tmp_path / "b") + run( + system.execute( + "gmail", + "send_gmail", + {"subject": "s", "body": "x", "inline_images": [first, second]}, + ) + ) + cids = [p["Content-ID"] for p in sent[0].walk() if p.get_content_maintype() == "image"] + assert cids == ["", ""] + + +@pytest.mark.parametrize("name", ["missing.png", "notes.txt"]) +def test_bad_inline_image_fails_without_sending(system, sent, tmp_path, name): + if name == "notes.txt": + (tmp_path / name).write_text("not an image") + result = run( + system.execute( + "gmail", + "send_gmail", + {"subject": "s", "body": "x", "inline_images": [str(tmp_path / name)]}, + ) + ) + assert result["status"] == "error" + assert name in result["message"] + assert sent == [] + + +def test_reply_and_draft_support_inline_images(system, sent, tmp_path, monkeypatch): + path = _image(tmp_path) + monkeypatch.setattr( + BoundGmailClient, + "_fetch_reply_headers", + lambda self, mid: {"From": "c@z.com", "Subject": "Hi", "_thread_id": "t1"}, + ) + body = {"body": '', "html": True, "inline_images": [path]} + run(system.execute("gmail", "reply_gmail", {"message_id": "m0", **body})) + run( + system.execute( + "gmail", "create_gmail_draft", {"to": "c@z.com", "subject": "s", **body} + ) + ) + assert len(sent) == 2 + for msg in sent: + assert _parts(msg)["image/png"]["Content-ID"] == "" + + def test_default_providers_importable(): from craftos_integrations.providers import default_providers From 236eff6959c2022e896581c0f63d7c94ef818646 Mon Sep 17 00:00:00 2001 From: zfoong Date: Mon, 5 Oct 2026 17:47:51 +0900 Subject: [PATCH 14/55] Sidebar UI update --- .../src/components/layout/NavBar.module.css | 52 ++-- .../frontend/src/components/layout/NavBar.tsx | 205 +++++++++---- .../browser/frontend/src/hooks/index.ts | 1 + .../frontend/src/hooks/useSortableList.ts | 283 ++++++++++++++++++ .../frontend/src/store/uiState/catalog.ts | 4 + 5 files changed, 469 insertions(+), 76 deletions(-) create mode 100644 app/ui_layer/browser/frontend/src/hooks/useSortableList.ts diff --git a/app/ui_layer/browser/frontend/src/components/layout/NavBar.module.css b/app/ui_layer/browser/frontend/src/components/layout/NavBar.module.css index cbae189c..b1ba3618 100644 --- a/app/ui_layer/browser/frontend/src/components/layout/NavBar.module.css +++ b/app/ui_layer/browser/frontend/src/components/layout/NavBar.module.css @@ -394,10 +394,8 @@ /* Main-row ⓘ hint. The label gives up its flex growth so the icon hugs the "Main" text; margin-right:auto on the hint keeps the status dot flush - right. The tooltip anchors to .sessionRow (nearest positioned ancestor) - and spans the row width (the sidebar clips horizontal overflow, so it - must not poke past the row's edges). display:none when hidden so the - absolutely-positioned box never inflates the sidebar's scroll height. */ + right. Hovering it shows .mainInfoTooltip, portalled to by + SidebarPopover so the sidebar's edges and bottom toolbar can't clip it. */ .sessionRowMain .label { flex: 0 1 auto; } @@ -416,12 +414,11 @@ } .mainInfoTooltip { - display: none; - position: absolute; - top: calc(100% + 2px); - left: var(--space-1); - right: var(--space-1); + position: fixed; z-index: var(--z-tooltip); + width: 280px; + /* Informational only; never steals the hover from the ⓘ icon. */ + pointer-events: none; padding: var(--space-2) var(--space-3); background: var(--bg-secondary); border: 1px solid var(--border-primary); @@ -433,11 +430,6 @@ color: var(--text-secondary); text-align: left; white-space: normal; - cursor: default; -} - -.mainInfo:hover .mainInfoTooltip { - display: block; } .mainInfoTooltip strong { @@ -482,12 +474,11 @@ background: var(--bg-selected); } -/* Context menu dropdown */ +/* Context menu dropdown. Portalled to and placed by SidebarPopover + (fixed, below its "…" button), so it can extend past the sidebar's edge. */ .sessionMenu { - position: absolute; - top: calc(100% - 2px); - right: var(--space-1); - z-index: 30; + position: fixed; + z-index: var(--z-dropdown); min-width: 210px; padding: var(--space-1); background: var(--bg-secondary); @@ -499,13 +490,6 @@ gap: 2px; } -/* Applied by positionSessionMenu() when the dropdown would overflow the - bottom of the sidebar scroll area: anchor above the row instead. */ -.sessionMenuUp { - top: auto; - bottom: calc(100% - 2px); -} - .sessionMenuItem { display: flex; align-items: center; @@ -530,6 +514,22 @@ color: var(--color-error); } +/* Row being dragged to a new spot (useSortableList): lifted and tilted above + its neighbours, which slide aside to open the drop slot. The tilt uses the + standalone `rotate` property, so it eases in on its own while the inline + `transform` keeps the row glued to the pointer. */ +.rowDragging { + background: var(--bg-secondary); + box-shadow: 0 6px 18px rgba(0, 0, 0, 0.3); + rotate: 2deg; + transition: rotate 120ms ease, box-shadow 120ms ease; +} + +.rowDragging, +.rowDragging * { + cursor: grabbing; +} + /* Inline rename input */ .renameInput { flex: 1 1 auto; diff --git a/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx b/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx index 773d09f7..b5a20867 100644 --- a/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx +++ b/app/ui_layer/browser/frontend/src/components/layout/NavBar.tsx @@ -31,7 +31,7 @@ import { useTheme } from '../../contexts/ThemeContext' import { removeIframe } from '../../pages/AgentApp/iframePool' import { ConfirmModal } from '../ui/ConfirmModal' import { tourAnchorProps, useTourEnvAction, type TourAnchorId } from '../../tour' -import { usePersistedState, useSkillCreator } from '../../hooks' +import { applySavedOrder, usePersistedState, useSkillCreator, useSortableList } from '../../hooks' import { CreateAgentAppModal } from '../ui/CreateAgentAppModal' import { SkillCreatorModal } from '../ui/SkillCreatorModal' import { AgentAppIcon } from '../ui/AgentAppIcon' @@ -95,14 +95,63 @@ function AnimatedSessionTitle({ title }: { title: string }) { ) } +// Popup anchored to a sidebar control, portalled to so it can extend +// past the sidebar's edge instead of being clipped by its scroll area. Opens +// below the anchor, flips above it when the viewport has no room below, and +// shifts left if it would run off the right edge. +function SidebarPopover({ + anchor, + className, + role, + children, +}: { + anchor: DOMRect + className: string + role: 'menu' | 'tooltip' + children: React.ReactNode +}) { + const ref = useRef(null) + const [pos, setPos] = useState<{ top: number; left: number } | null>(null) + + useLayoutEffect(() => { + const el = ref.current + if (!el) return + const { width, height } = el.getBoundingClientRect() + const margin = 8 + const below = anchor.bottom + 4 + const top = below + height <= window.innerHeight - margin + ? below + : Math.max(margin, anchor.top - 4 - height) + const left = Math.max(margin, Math.min(anchor.left, window.innerWidth - margin - width)) + setPos({ top, left }) + }, [anchor]) + + return createPortal( + , + document.body, + ) +} + interface NavBarProps { collapsed?: boolean onToggleCollapsed?: () => void } -// Per-row "…" context menu state: which session's menu is open. +// Per-row "…" context menu state: which session's menu is open, and where. interface SessionMenuState { sessionId: string + /** The "…" button's rect; the menu opens below it. */ + anchor: DOMRect } // How many Agent App items show before the "Show more" row takes over. @@ -135,18 +184,31 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { renameSession, clearSession, } = useWebSocket() - const agentAppProjects = useAppSelector(selectAgentAppProjects) + // Both lists follow the user's drag-and-drop order. Items created since the + // last reorder land where they always have: new apps last, new chats first. + const [agentAppOrder, setAgentAppOrder] = usePersistedState(UI_STATE.nav.agentAppOrder) + const [chatOrder, setChatOrder] = usePersistedState(UI_STATE.nav.chatOrder) + const unorderedAgentAppProjects = useAppSelector(selectAgentAppProjects) + const agentAppProjects = useMemo( + () => applySavedOrder(unorderedAgentAppProjects, agentAppOrder, p => p.id, 'last'), + [unorderedAgentAppProjects, agentAppOrder], + ) const skillMeta = useAppSelector(selectSkillMeta) const [lastSeenBySession] = usePersistedState(UI_STATE.chat.lastSeenMessageIds) const { theme } = useTheme() const [showCreateModal, setShowCreateModal] = useState(false) - // Which Agent App row's "…" menu is open, and the app queued for a delete - // confirmation (delete is destructive, so it always confirms). - const [agentMenu, setAgentMenu] = useState(null) + // Which Agent App row's "…" menu is open (and its button's rect), and the + // app queued for a delete confirmation (delete is destructive, so it always + // confirms). + const [agentMenu, setAgentMenu] = useState<{ id: string; anchor: DOMRect } | null>(null) const [agentDelete, setAgentDelete] = useState<{ id: string; name: string } | null>(null) const mainSession = useAppSelector(selectMainSession) - const chatSessions = useAppSelector(selectChatSessions) + const unorderedChatSessions = useAppSelector(selectChatSessions) + const chatSessions = useMemo( + () => applySavedOrder(unorderedChatSessions, chatOrder, s => s.id, 'first'), + [unorderedChatSessions, chatOrder], + ) const lastMessageIdBySession = useAppSelector(selectLastMessageIdBySession) const runStateBySession = useAppSelector(state => state.agent.runStateBySession) @@ -208,6 +270,8 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { }, [collapsed]) const [menu, setMenu] = useState(null) + // The Main row's ⓘ icon rect while it's hovered (shows the tooltip). + const [mainTip, setMainTip] = useState(null) const [renamingId, setRenamingId] = useState(null) const [renameDraft, setRenameDraft] = useState('') const renameInputRef = useRef(null) @@ -227,6 +291,29 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { const [canScrollUp, setCanScrollUp] = useState(false) const [canScrollDown, setCanScrollDown] = useState(false) + const visibleAgentAppProjects = showAllAgentApp + ? agentAppProjects + : agentAppProjects.slice(0, GROUP_PREVIEW_COUNT) + const visibleChatSessions = chatSessions.slice(0, chatVisibleCount) + + // Drag-to-reorder within each group. The rendered rows are always a prefix + // of the full list ("Show more" / scroll pagination), so the reordered + // prefix followed by the untouched tail is the new full order. + const withTail = (prefix: string[], all: string[]) => { + const moved = new Set(prefix) + return [...prefix, ...all.filter(id => !moved.has(id))] + } + const agentAppSortable = useSortableList({ + ids: visibleAgentAppProjects.map(p => p.id), + onReorder: ids => setAgentAppOrder(withTail(ids, agentAppProjects.map(p => p.id))), + scrollRef, + }) + const chatSortable = useSortableList({ + ids: visibleChatSessions.map(s => s.id), + onReorder: ids => setChatOrder(withTail(ids, chatSessions.map(s => s.id))), + scrollRef, + }) + const isActive = (path: string) => { if (path === '/') { return location.pathname === '/' @@ -428,26 +515,31 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { } }, []) - // Flip the "…" menu upward when opening downward would spill past the - // visible bottom of the sidebar scroll area — otherwise the menu just - // grows the scroll height and the user has to scroll to reach it. - const positionSessionMenu = (el: HTMLDivElement | null) => { - if (!el) return - const container = scrollRef.current - const limit = container - ? Math.min(container.getBoundingClientRect().bottom, window.innerHeight) - : window.innerHeight - if (el.getBoundingClientRect().bottom > limit - 4) { - el.classList.add(styles.sessionMenuUp) + // Popovers are placed once, in viewport coordinates, so they'd drift off + // their row if the sidebar scrolled or the window resized: close them. + const popoverOpen = menu !== null || agentMenu !== null || mainTip !== null + useEffect(() => { + if (!popoverOpen) return + const el = scrollRef.current + const close = () => { + setMenu(null) + setAgentMenu(null) + setMainTip(null) } - } + el?.addEventListener('scroll', close) + window.addEventListener('resize', close) + return () => { + el?.removeEventListener('scroll', close) + window.removeEventListener('resize', close) + } + }, [popoverOpen]) // "…" context menu attached to a session row. `isMain` limits the menu // to Clear conversation + Create skill for the pinned Main session. const renderSessionMenu = (session: SessionInfo, isMain: boolean) => { if (menu?.sessionId !== session.id) return null return ( -
e.stopPropagation()}> + {!isMain && ( -
+ ) } @@ -475,11 +567,16 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { const path = sessionPath(session.id) const active = isActive(path) const renaming = renamingId === session.id + // Main stays pinned first; every other chat can be dragged. + const sortable = opts.isMain ? null : chatSortable + const dragging = chatSortable.draggingId === session.id return (
@@ -501,6 +598,7 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { className={styles.sessionRowButton} onClick={() => navigate(path)} title={opts.isMain ? t('nav:items.main') : session.title} + {...sortable?.handleProps(session.id)} > {opts.isMain ? : } @@ -509,20 +607,28 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { {opts.isMain ? t('nav:items.main') : } {opts.isMain && ( - + setMainTip(e.currentTarget.getBoundingClientRect())} + onMouseLeave={() => setMainTip(null)} + > - - {t('nav:mainTooltip.title')} - - {t('nav:mainTooltip.line1')} - - - {t('nav:mainTooltip.line2')} - - - {t('nav:mainTooltip.line3')} - - + {mainTip && ( + + {t('nav:mainTooltip.title')} + + {t('nav:mainTooltip.line1')} + + + {t('nav:mainTooltip.line2')} + + + {t('nav:mainTooltip.line3')} + + + )} )} {renderSessionDot(session.id)} @@ -531,10 +637,11 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { className={styles.sessionMenuButton} onClick={(e) => { e.stopPropagation() + const anchor = e.currentTarget.getBoundingClientRect() setMenu(prev => prev?.sessionId === session.id ? null - : { sessionId: session.id }) + : { sessionId: session.id, anchor }) }} aria-label={t('nav:sessionMenu.sessionOptions')} title={t('nav:sessionMenu.options')} @@ -578,17 +685,21 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) { // bridge — so it's only offered for the active app. const showTheme = active const showKebab = lifecycle !== null || showTheme || canDelete + const dragging = agentAppSortable.draggingId === project.id return (
)} - {agentMenu === project.id && showKebab && ( -
e.stopPropagation()} - > + {agentMenu?.id === project.id && showKebab && ( + {lifecycle === 'stop' && ( )} -
+ )}
) @@ -804,10 +912,7 @@ export function NavBar({ collapsed = false, onToggleCollapsed }: NavBarProps) {
{agentAppExpanded && (
- {(showAllAgentApp - ? agentAppProjects - : agentAppProjects.slice(0, GROUP_PREVIEW_COUNT) - ).map(project => renderAgentAppRow(project))} + {visibleAgentAppProjects.map(project => renderAgentAppRow(project))} {agentAppProjects.length > GROUP_PREVIEW_COUNT && (
diff --git a/app/ui_layer/browser/frontend/src/hooks/index.ts b/app/ui_layer/browser/frontend/src/hooks/index.ts index 105a1793..6ad974a7 100644 --- a/app/ui_layer/browser/frontend/src/hooks/index.ts +++ b/app/ui_layer/browser/frontend/src/hooks/index.ts @@ -7,4 +7,5 @@ export { useScrollRestoration } from './useScrollRestoration' export { useServerDraft } from './useServerDraft' export type { ServerDraft } from './useServerDraft' export { useSkillCreator } from './useSkillCreator' +export { useSortableList, applySavedOrder } from './useSortableList' export type { SkillCreatorStatus } from './useSkillCreator' diff --git a/app/ui_layer/browser/frontend/src/hooks/useSortableList.ts b/app/ui_layer/browser/frontend/src/hooks/useSortableList.ts new file mode 100644 index 00000000..fdfac733 --- /dev/null +++ b/app/ui_layer/browser/frontend/src/hooks/useSortableList.ts @@ -0,0 +1,283 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type CSSProperties, + type DragEvent as ReactDragEvent, + type PointerEvent as ReactPointerEvent, + type RefObject, +} from 'react' + +/** Pointer travel (px) before a press on a row turns into a drag. */ +const DRAG_THRESHOLD = 5 +/** Distance (px) from the scroll container's edge where auto-scroll kicks in. */ +const AUTO_SCROLL_EDGE = 36 +/** Max auto-scroll speed in px per frame, reached at the very edge. */ +const AUTO_SCROLL_SPEED = 12 + +interface DragState { + id: string + /** How far the dragged row has travelled, in content px (includes auto-scroll). */ + offset: number + /** Index of the dragged row in the drag-start snapshot. */ + from: number + /** Index it would drop at. */ + to: number + /** Distance between adjacent rows: the dragged row's height plus the list gap. */ + pitch: number + /** Ids at drag start; rows mounted mid-drag (pagination) are left in place. */ + ids: string[] +} + +interface SortableListOptions { + /** Ids of the rendered rows, in display order. */ + ids: string[] + /** Called on drop with the rendered ids in their new order. */ + onReorder: (ids: string[]) => void + /** The scrolling ancestor, auto-scrolled while dragging near its edges. */ + scrollRef: RefObject +} + +/** + * Drag-to-reorder for a vertical list, built on pointer events. (Native HTML5 + * drag and drop won't start from a