diff --git a/DECISIONS.md b/DECISIONS.md index e86b1844..7fe5efdd 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -19839,3 +19839,70 @@ name as the owner of its windows, so a name check missed exactly the copy TST Desk ships. The executable check must not catch TST Desk's own windows, which share that owner name. The lock covers set-up only, never the command, so concurrent sessions still act in parallel. + +## 2026-11-04 — TD-5233: a thinking-only reply at the output cap is asked once more with a larger cap (Class B) + +**Decision:** when a turn's model call ends with `finish_reason: length`, no +text and no tool call, the loop withdraws that response (`assistant_retract`, +reason `output_cap_thinking`) and asks once more under a new response id with +a larger `max_tokens`. There is no backoff wait, since the endpoint did not +fail, but Stop and cancel are checked after the withdrawal. + +Protocol (additive, no version bump): `turn_complete.error_code` gains +`output_cap_thinking`, and `assistant_retract.reason` gains the same string. +An older window shows the code as an unrecognised error and the retract as +"Provider went quiet". The spec is silent on both; the TD-5233 criteria set +them. + +1. **The raised cap.** `min(cap × 4, max(32768, cap), room)`, where room is + `context_window` minus the prompt estimate minus a margin of + `max(1024, context_window / 20)`. It is sent only when it is at least + twice the cap; otherwise there is no retry and the turn fails as below. + A 4096 cap on a 256k window with a 28k prompt becomes 16384. +2. **One retry per model call, on its own counter.** The stall, cut-off and + reply-guard retries keep their own allowance. `provider_retry.max_retries: + 0` turns this retry off, as it does the stall retry. Once a call's cap is + raised, its later attempts keep it, so a stall retry after the cap retry + does not drop back to the tier's cap. +3. **A refusal does not hide the cause.** If the raised request fails before + any output with an error that is not retried, the turn reports the first, + empty-at-cap result and a warning names the refusal's code. Only the call + right after the cap retry counts as the raised request: a refusal of a + later stall or guard retry is that call's own error and is reported as it + is. OpenRouter + reserves `max_tokens` against the account balance, so a 402 there, or a + 400 or a context overflow, is caused by the retry itself, and reporting it + would replace the real problem. A retryable failure is returned as it is, + so the usual retry and backup handling applies. +4. **No reasoning fields.** No OpenRouter `reasoning` field is sent to limit + or drop the thinking. `effort.py` sends reasoning fields only to tiers that + opt in, and whether DeepSeek honours them through OpenRouter is unverified. +5. **No spend-cap check before the retry.** The cap is checked before each + model call, not before an in-call retry, as for a stall retry. The 32768 + ceiling bounds what the one extra request can spend. +6. **What the chat says.** A turn that still has no answer at the cap fails + as `output_cap_thinking`. The line names the cap (and the retry's cap when + there was one) and the config entry whose value applied, as its + `presets..` path: a backup names its `fallbacks[n]` entry + and a remapped computer-use worker names the `local_worker_preset`'s + worker, because raising the tier's own value would not change either. It + says to raise `max_output_tokens` there in Settings → Configuration. It says new chats use the new limit because an + open session keeps the config snapshot it started with. An empty reply that + ended any other way stays `empty_completion` and says to resend. Either + line suggests a new chat only when the prompt estimate is at or past the + compaction point. The lines are short because they enter the conversation + and the model reads them on the next turn. + +**Rationale:** OpenRouter and most providers count thinking tokens inside +`max_tokens`, so a reasoning model on a small cap can spend all of it before +writing a word. The old line blamed the context, which was a tenth full, and +nothing asked again, so the next turns failed the same way. Four times the +cap gives the thinking real room on one extra request, and the ceiling and +margin keep that request bounded and inside the window. Under twice the cap, +a retry is unlikely to finish where the first did not. + +**Rejected:** retrying at the same cap (the model thinks the same way and +ends the same way); writing a larger cap into the tier's config (that is the +user's setting, and the line says where to change it); more than one cap +retry (each is a long, paid call); limiting the thinking with OpenRouter's +`reasoning` fields (point 4). diff --git a/core/tests/test_thinking_cap_retry.py b/core/tests/test_thinking_cap_retry.py new file mode 100644 index 00000000..a420648e --- /dev/null +++ b/core/tests/test_thinking_cap_retry.py @@ -0,0 +1,669 @@ +"""A reply that spent the whole output cap thinking (TD-5233). + +On most providers a reasoning model's thinking counts against +``max_tokens``. A 4096-token cap could go entirely on thinking: the stream +ended ``finish_reason: "length"`` with no answer and no tool call, the turn +failed as ``empty_completion`` with advice to start a new session (wrong at +28k of a 256k window), and nothing was retried. Now the reply is withdrawn +and asked once more with a larger cap, and a turn that still gets no answer +says which setting to raise. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from tests.test_cap_enforcement import make_echo_session +from tests.test_dispatch import start_loop as start_dispatch_loop +from tests.test_loop import start_loop, wait_for_turn +from tests.test_reply_guard import _BodyGateway, _varied +from tests.test_turn_stream_errors import _DONE, _answer, _event, _text +from tstd.boundary_config import BoundaryConfig +from tstd.config import ModelConfig, Preset, ProviderRetryConfig, ReplyGuardConfig, TierConfig +from tstd.loop import agent_loop +from tstd.mock import MockProvider, Script +from tstd.output_cap import cap_setting, cut_call_note, raised_output_cap +from tstd.protocol import ( + AssistantReasoning, + AssistantRetract, + ContextCompacted, + DaemonEvent, + TurnComplete, +) +from tstd.provider import ChatCompletionRequest, ProviderClient, TimeoutConfig +from tstd.reply_retry import StreamResult, shown_deltas, stream_reply, thinking_ate_cap +from tstd.router import TierRouter +from tstd.session import Session, SessionRunner + +_CAP = 4_096 +_WINDOW = 262_144 +_RAISED = 16_384 +_ANSWER = "Use the second option." +_NO_ANSWER = "No answer: the model ended its turn without writing anything. Resend to try again." +_CONTEXT_NOTE = "The conversation is also close to its context limit, so a new chat may help." + + +def _tier(slug: str, *, cap: int, window: int, guard: ReplyGuardConfig | None) -> TierConfig: + return TierConfig( + slug=slug, + base_url="http://mock.local/v1", + input_price=1.0, + output_price=2.0, + cache_read_price=0.5, + context_window=window, + max_output_tokens=cap, + reply_guard=guard, + ) + + +def _config( + *, + cap: int = _CAP, + window: int = _WINDOW, + retries: int = 3, + guard: ReplyGuardConfig | None = None, +) -> ModelConfig: + config = ModelConfig( + presets={ + "test": Preset( + brain=_tier("test-brain", cap=cap, window=window, guard=guard), + worker=_tier("test-worker", cap=cap, window=window, guard=None), + validator=_tier("test-validator", cap=cap, window=window, guard=None), + ), + }, + active_preset="test", + ) + config.provider_retry = ProviderRetryConfig( + max_retries=retries, initial_delay=0.001, max_delay=0.001 + ) + return config + + +def _thinks_to_cap() -> Script: + """A reasoning model that spends the whole cap thinking.""" + return Script(kind="stream", reasoning="Weighing every option first", finish_reason="length") + + +async def _run( + tmp_path: Path, + scripts: list[Script], + config: ModelConfig, + *, + message: str = "Which option?", + stop_on_retract: bool = False, +) -> tuple[Session, TurnComplete, MockProvider]: + session = Session(str(tmp_path)) + mock = MockProvider( + sequences={"test-brain": scripts}, + scripts={"test-brain": Script(kind="stream", content="UNEXPECTED EXTRA CALL")}, + ) + runner = await start_loop(session, TierRouter(), mock, config) + try: + await session.add_user_message(message) + if stop_on_retract: + await _until(lambda: bool(_of(session, AssistantRetract))) + await session.stop_turn() + turn = await wait_for_turn(session, 1) + finally: + await runner.cancel() + return session, turn, mock + + +async def _until(check: Any) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + 3.0 + while loop.time() < deadline: + if check(): + return + await asyncio.sleep(0.005) + raise TimeoutError("condition not met") + + +def _of(session: Session, kind: type[Any]) -> list[Any]: + return [e for e in session.event_log.all_events if isinstance(e, kind)] + + +def _brain_calls(mock: MockProvider) -> list[ChatCompletionRequest]: + return [call for call in mock.calls if call.model == "test-brain"] + + +def _caps(mock: MockProvider) -> list[int | None]: + return [call.max_tokens for call in _brain_calls(mock)] + + +def _shown(session: Session) -> str: + return "".join(shown_deltas(session.event_log.all_events)) + + +def _last_assistant(session: Session) -> str: + return str([m for m in session.conversation if m.role == "assistant"][-1].content) + + +# ── The retry ─────────────────────────────────────────────────────────── + + +async def test_a_reply_that_thought_to_the_cap_is_asked_again_with_a_larger_cap( + tmp_path: Path, +) -> None: + session, turn, mock = await _run( + tmp_path, [_thinks_to_cap(), Script(kind="stream", content=_ANSWER)], _config() + ) + + # Before TD-5233: one call, and a failed empty_completion turn. + assert (turn.failed, turn.error_code) == (False, None) + assert _caps(mock) == [_CAP, _RAISED] + (retract,) = _of(session, AssistantRetract) + assert retract.reason == "output_cap_thinking" + first_thought = _of(session, AssistantReasoning)[0] + assert retract.response_id == first_thought.response_id + assert _shown(session) == _ANSWER + assert _last_assistant(session) == _ANSWER + + +async def test_both_attempts_at_the_cap_fail_naming_the_setting(tmp_path: Path) -> None: + session, turn, mock = await _run(tmp_path, [_thinks_to_cap(), _thinks_to_cap()], _config()) + + assert (turn.failed, turn.error_code) == (True, "output_cap_thinking") + assert _caps(mock) == [_CAP, _RAISED] + assert len(_of(session, AssistantRetract)) == 1 + text = _last_assistant(session) + assert text == ( + "No answer: the model used its whole output limit thinking, including one retry at " + "16,384 tokens. Raise max_output_tokens for the brain tier (presets.test.brain), now " + "4,096, in Settings → Configuration. New chats use the new limit." + ) + # The context is far from full, so starting over is not the advice. + assert "new chat may help" not in text and "new session" not in text.lower() + assert _shown(session) == text + + +async def test_an_empty_reply_not_at_the_cap_is_not_retried(tmp_path: Path) -> None: + thinks_then_stops = Script(kind="stream", reasoning="Hmm, well") + session, turn, mock = await _run(tmp_path, [thinks_then_stops], _config()) + + assert (turn.failed, turn.error_code) == (True, "empty_completion") + assert len(_brain_calls(mock)) == 1 + assert not _of(session, AssistantRetract) + assert _last_assistant(session) == _NO_ANSWER + + +async def test_max_retries_zero_turns_it_off(tmp_path: Path) -> None: + session, turn, mock = await _run(tmp_path, [_thinks_to_cap()], _config(retries=0)) + + assert (turn.failed, turn.error_code) == (True, "output_cap_thinking") + assert _caps(mock) == [_CAP] + assert not _of(session, AssistantRetract) + assert _last_assistant(session) == ( + "No answer: the model used its whole output limit (4,096 tokens) thinking. Raise " + "max_output_tokens for the brain tier (presets.test.brain) in Settings → " + "Configuration. New chats use the new limit." + ) + + +@pytest.mark.parametrize( + ("code", "status"), [("insufficient_credits", 402), ("context_overflow", 400)] +) +async def test_a_refused_larger_cap_keeps_the_first_reply( + tmp_path: Path, caplog: pytest.LogCaptureFixture, code: str, status: int +) -> None: + refused = Script(kind="error", error_code=code, status_code=status, content="refused") + with caplog.at_level(logging.WARNING, logger="tstd.reply_retry"): + session, turn, mock = await _run(tmp_path, [_thinks_to_cap(), refused], _config()) + + # The cause is the cap, not the refusal of the larger one. + assert (turn.failed, turn.error_code) == (True, "output_cap_thinking") + assert _caps(mock) == [_CAP, _RAISED] + assert "(4,096 tokens)" in _last_assistant(session) + # No tighter retry for an overflow, and nothing elided for one. + assert not _of(session, ContextCompacted) + (warning,) = [r for r in caplog.records if "larger output cap was refused" in r.getMessage()] + assert warning.extra_fields["code"] == code # type: ignore[attr-defined] + + +async def test_a_retryable_failure_of_the_larger_cap_is_the_turns(tmp_path: Path) -> None: + down = Script(kind="error", status_code=503, content="upstream unavailable") + _, turn, mock = await _run(tmp_path, [_thinks_to_cap(), down], _config()) + + assert (turn.failed, turn.error_code) == (True, "server_error") + assert len(_brain_calls(mock)) == 2 + + +async def test_no_retry_when_the_window_has_no_room_for_double(tmp_path: Path) -> None: + session, turn, mock = await _run(tmp_path, [_thinks_to_cap()], _config(window=9_000)) + + assert (turn.failed, turn.error_code) == (True, "output_cap_thinking") + assert _caps(mock) == [_CAP] + assert not _of(session, AssistantRetract) + assert "(4,096 tokens)" in _last_assistant(session) + + +async def test_the_larger_cap_is_bounded_by_the_room_in_the_window(tmp_path: Path) -> None: + window = 14_000 + _, turn, mock = await _run( + tmp_path, + [_thinks_to_cap(), Script(kind="stream", content=_ANSWER)], + _config(window=window), + ) + + assert not turn.failed + first, raised = _caps(mock) + assert first == _CAP + assert raised is not None + assert 2 * _CAP <= raised < 4 * _CAP + assert raised <= window - 1_024 + + +async def test_stop_during_the_retry_ends_the_turn_as_stopped(tmp_path: Path) -> None: + slow = Script(kind="stream", content="An answer that streams slowly", chunk_delay=0.2) + session, turn, _ = await _run( + tmp_path, [_thinks_to_cap(), slow], _config(), stop_on_retract=True + ) + + assert (turn.failed, turn.error_code) == (False, "stopped") + assert not any("No answer" in str(m.content) for m in session.conversation) + assert _shown(session) == "" + + +class _StopsAsItEnds(MockProvider): + """Stop lands after the last chunk, before the loop reads the reply.""" + + def __init__(self, session: Session, script: Script) -> None: + super().__init__(scripts={"test-brain": script}) + self._session = session + + async def chat_completion_stream(self, request: ChatCompletionRequest) -> AsyncIterator[Any]: + async for item in super().chat_completion_stream(request): + yield item + await self._session.stop_turn() + + +async def test_stop_as_the_reply_ends_is_not_a_failed_turn(tmp_path: Path) -> None: + session = Session(str(tmp_path)) + mock = _StopsAsItEnds(session, _thinks_to_cap()) + runner = await start_loop(session, TierRouter(), mock, _config()) + try: + await session.add_user_message("Which option?") + turn = await wait_for_turn(session, 1) + finally: + await runner.cancel() + + assert (turn.failed, turn.error_code) == (False, "stopped") + assert len(_brain_calls(mock)) == 1 + assert not _of(session, AssistantRetract) + assert not any("No answer" in str(m.content) for m in session.conversation) + + +async def test_a_cut_tool_call_at_the_larger_cap_names_that_cap( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + session, dispatcher = make_echo_session(tmp_path, BoundaryConfig()) + cut = Script( + kind="tool_call", + tool_batch=(("echo", '{"message": "first"}'), ("echo", '{"message": "seco')), + finish_reason="length", + ) + mock = MockProvider( + sequences={ + "test-brain": [_thinks_to_cap(), cut, Script(kind="stream", content="Sent one.")] + } + ) + runner = await start_dispatch_loop( + session, TierRouter(lead_turns=80), mock, _config(), None, dispatcher + ) + try: + with caplog.at_level(logging.WARNING, logger="tstd.output_cap"): + await session.add_user_message("echo twice") + turn = await wait_for_turn(session, 1) + finally: + await runner.cancel() + + assert not turn.failed + replay = _brain_calls(mock)[-1].messages + replies = {m.tool_call_id: m.content for m in replay if m.role == "tool"} + assert replies["call_mock_2"] == cut_call_note(_RAISED) + caps = [ + r.extra_fields["max_output_tokens"] # type: ignore[attr-defined] + for r in caplog.records + if r.getMessage() == "reply hit the output cap" + ] + assert caps == [_RAISED] + + +# ── Near the context limit ────────────────────────────────────────────── + + +async def test_a_backup_that_thought_to_its_cap_names_its_own_setting(tmp_path: Path) -> None: + """The backup's own max_output_tokens applied, so the copy names it (TD-5233).""" + config = _config() + backup = _tier("test-backup", cap=8_192, window=_WINDOW, guard=None) + config.presets["test"].brain.fallbacks = [backup] + down = Script(kind="error", status_code=503, content="upstream unavailable") + session = Session(str(tmp_path)) + mock = MockProvider( + sequences={"test-brain": [down], "test-backup": [_thinks_to_cap(), _thinks_to_cap()]}, + ) + runner = await start_loop(session, TierRouter(), mock, config) + try: + await session.add_user_message("Which option?") + turn = await wait_for_turn(session, 1) + finally: + await runner.cancel() + + assert (turn.failed, turn.error_code) == (True, "output_cap_thinking") + assert [(c.model, c.max_tokens) for c in mock.calls if c.model == "test-backup"] == [ + ("test-backup", 8_192), + ("test-backup", 32_768), + ] + assert _last_assistant(session) == ( + "No answer: the model used its whole output limit thinking, including one retry at " + "32,768 tokens. Raise max_output_tokens for the brain tier's backup " + "(presets.test.brain.fallbacks[0]), now 8,192, in Settings → Configuration. New chats " + "use the new limit." + ) + + +@pytest.mark.parametrize( + ("scripts", "code", "sent", "finish"), + [ + ([_thinks_to_cap(), _thinks_to_cap()], "output_cap_thinking", _RAISED, "length"), + ([Script(kind="stream", reasoning="Hmm")], "empty_completion", _CAP, "stop"), + ], +) +async def test_the_empty_reply_warning_names_model_cap_reason_prompt_and_code( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + scripts: list[Script], + code: str, + sent: int, + finish: str, +) -> None: + with caplog.at_level(logging.WARNING, logger="tstd.loop"): + await _run(tmp_path, scripts, _config()) + + (record,) = [r for r in caplog.records if r.getMessage() == "turn failed: empty completion"] + fields = record.extra_fields # type: ignore[attr-defined] + assert fields["model"] == "test-brain" + assert fields["max_tokens"] == sent + assert fields["finish_reason"] == finish + assert fields["error_code"] == code + assert isinstance(fields["prompt_tokens"], int) and fields["prompt_tokens"] > 0 + + +@pytest.mark.parametrize( + ("script", "code"), + [ + (Script(kind="stream", reasoning="Hmm", finish_reason="length"), "output_cap_thinking"), + (Script(kind="stream", reasoning="Hmm"), "empty_completion"), + ], +) +async def test_a_near_full_context_adds_the_new_chat_note( + tmp_path: Path, script: Script, code: str +) -> None: + # A single turn cannot be compacted, so the prompt stays past the + # budget (0.8 of 7000) and leaves no room for a larger cap. + session, turn, mock = await _run( + tmp_path, [script], _config(cap=1_000, window=8_000), message="hi " + "word " * 4_600 + ) + + assert (turn.failed, turn.error_code) == (True, code) + assert len(_brain_calls(mock)) == 1 + assert _last_assistant(session).endswith(" " + _CONTEXT_NOTE) + + +async def test_the_note_is_left_out_when_the_context_is_not_near_full(tmp_path: Path) -> None: + session, _, _ = await _run( + tmp_path, [Script(kind="stream", reasoning="Hmm")], _config(cap=1_000, window=8_000) + ) + assert _last_assistant(session) == _NO_ANSWER + + +# ── The larger cap sticks for the rest of the call ────────────────────── + +_IDLE = TimeoutConfig(stream_idle=0.2) + + +def _thinking(text: str) -> bytes: + return _event({"id": "c1", "choices": [{"index": 0, "delta": {"reasoning_content": text}}]}) + + +def _thought_to_cap(text: str = "Weighing it.") -> bytes: + return _thinking(text) + _text("", finish="length") + _DONE + + +async def _stalls_after(*chunks: bytes) -> AsyncIterator[bytes]: + for chunk in chunks: + yield chunk + await asyncio.sleep(5) + + +async def _run_gateway( + tmp_path: Path, gateway: _BodyGateway, config: ModelConfig +) -> tuple[Session, TurnComplete]: + session = Session(str(tmp_path)) + client: ProviderClient = gateway.client(retries=2, timeout=_IDLE) + + async def factory(*_args: object) -> ProviderClient: + return client + + runner = SessionRunner( + session, loop_factory=lambda s: agent_loop(s, TierRouter(), factory, config) + ) + await runner.start() + try: + await session.add_user_message("Which option?") + turn = await wait_for_turn(session, 1) + finally: + await runner.cancel() + await client.close() + return session, turn + + +async def test_a_stall_after_the_cap_retry_keeps_the_larger_cap(tmp_path: Path) -> None: + def reply(attempt: int) -> bytes | AsyncIterator[bytes]: + if attempt == 1: + return _thought_to_cap() + if attempt == 2: + return _stalls_after(_thinking("Now the second")) + return _answer(_ANSWER) + + gateway = _BodyGateway(reply) + session, turn = await _run_gateway(tmp_path, gateway, _config()) + + assert not turn.failed + assert [body["max_tokens"] for body in gateway.bodies] == [_CAP, _RAISED, _RAISED] + assert [r.reason for r in _of(session, AssistantRetract)] == [ + "output_cap_thinking", + "stream_stalled", + ] + assert _shown(session) == _ANSWER + + +async def test_the_larger_cap_composes_with_the_guards_thinking_off_retry( + tmp_path: Path, +) -> None: + def reply(attempt: int) -> bytes | AsyncIterator[bytes]: + if attempt == 1: + return _thought_to_cap() + if attempt == 2: + return _thinking(_varied("Thinking about it.", 60)) + _DONE + return _answer(_ANSWER) + + guard = ReplyGuardConfig( + max_reasoning_chars=400, + retry_extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + ) + gateway = _BodyGateway(reply) + _, turn = await _run_gateway(tmp_path, gateway, _config(guard=guard)) + + assert not turn.failed + assert [body["max_tokens"] for body in gateway.bodies] == [_CAP, _RAISED, _RAISED] + assert "chat_template_kwargs" not in gateway.bodies[1] + assert gateway.bodies[2]["chat_template_kwargs"] == {"enable_thinking": False} + + +# ── stream_reply on its own ───────────────────────────────────────────── + + +class _Log: + def __init__(self, session: _FakeSession) -> None: + self._session = session + self.events: list[DaemonEvent] = [] + + async def add(self, event: DaemonEvent) -> None: + self.events.append(event) + if isinstance(event, AssistantRetract) and self._session.stop_on_retract: + self._session.stop_turn_requested = True + + +class _FakeSession: + """What stream_reply reads of a session, with Stop pressed on demand.""" + + id = "s" + + def __init__(self, *, stop_on_retract: bool = False) -> None: + self.stop_on_retract = stop_on_retract + self.cancel_requested = False + self.stop_turn_requested = False + self.event_log = _Log(self) + + async def wait_for_turn_abort(self) -> None: + await asyncio.sleep(10) + + +def _result(**fields: Any) -> StreamResult: + base: dict[str, Any] = { + "content": "", + "tool_calls": {}, + "failed": False, + "error_msg": "", + "error_code": None, + "raw_detail": "", + "finish_reason": "length", + "reasoning_details": None, + "began": True, + "max_tokens": _CAP, + } + base.update(fields) + return StreamResult(**base) + + +async def _stream( + session: _FakeSession, results: list[StreamResult], *, cap_retries: int = 1 +) -> tuple[StreamResult, list[str]]: + ids: list[str] = [] + + async def attempt(response_id: str) -> StreamResult: + ids.append(response_id) + return results[len(ids) - 1]._replace(response_id=response_id) + + result = await stream_reply( + attempt, + session, # type: ignore[arg-type] + ProviderRetryConfig(max_retries=3, initial_delay=0.001, max_delay=0.001), + cap_retries=cap_retries, + raised_cap=_RAISED, + ) + return result, ids + + +async def test_stop_as_the_reply_is_withdrawn_keeps_nothing() -> None: + session = _FakeSession(stop_on_retract=True) + result, ids = await _stream(session, [_result(), _result(content=_ANSWER)]) + + assert len(ids) == 1 + assert (result.failed, result.error_msg, result.error_code) == (True, "stopped", None) + assert result.content == "" + + +async def test_the_first_reply_comes_back_when_the_larger_cap_is_refused() -> None: + session = _FakeSession() + first = _result(reasoning="Weighing it.") + refused = _result(failed=True, began=False, error_code="insufficient_credits") + result, ids = await _stream(session, [first, refused]) + + assert len(ids) == 2 + assert result == first._replace(response_id=ids[0]) + + +async def test_a_refusal_after_a_later_stall_retry_is_returned_as_itself() -> None: + """Only the larger request's refusal falls back to the first reply (TD-5233).""" + session = _FakeSession() + stalled = _result(failed=True, error_code="stream_stalled", max_tokens=_RAISED) + refused = _result(failed=True, began=False, error_code="auth_failed", max_tokens=_RAISED) + result, ids = await _stream(session, [_result(), stalled, refused]) + + assert len(ids) == 3 + assert (result.failed, result.error_code) == (True, "auth_failed") + + +async def test_without_an_allowance_the_reply_is_returned() -> None: + session = _FakeSession() + result, ids = await _stream(session, [_result()], cap_retries=0) + + assert len(ids) == 1 + assert thinking_ate_cap(result) + assert not session.event_log.events + + +# ── Unit ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("fields", "expected"), + [ + ({}, True), + ({"content": " \n\t"}, True), + ({"content": "An answer"}, False), + ({"tool_calls": {0: {"id": "c", "name": "fs_read", "arguments": "{}"}}}, False), + ({"failed": True, "error_code": "stream_stalled"}, False), + ({"finish_reason": "stop"}, False), + ({"finish_reason": None}, False), + ], +) +def test_thinking_ate_cap(fields: dict[str, Any], expected: bool) -> None: + assert thinking_ate_cap(_result(**fields)) is expected + + +@pytest.mark.parametrize( + ("cap", "window", "prompt", "expected"), + [ + # The incident: 4096 at 28k of 256k. + (4_096, 262_144, 28_000, 16_384), + # The ceiling, below four times the cap. + (12_288, 262_144, 1_000, 32_768), + (16_384, 262_144, 1_000, 32_768), + # The room, after a 1024 margin on a small window... + (4_096, 16_000, 2_000, 12_976), + # ...and a twentieth of a large one. + (4_096, 100_000, 80_000, 15_000), + # Exactly double is still worth asking. + (4_096, 10_216, 1_000, 8_192), + # Less than double is not. + (4_096, 11_000, 2_000, None), + (4_096, 8_000, 9_000, None), + # A cap at or past the ceiling has nowhere to go. + (32_768, 1_000_000, 0, None), + (65_536, 1_000_000, 0, None), + (20_000, 1_000_000, 0, None), + ], +) +def test_raised_output_cap(cap: int, window: int, prompt: int, expected: int | None) -> None: + assert raised_output_cap(cap, window, prompt) == expected + + +@pytest.mark.parametrize( + ("args", "expected"), + [ + (("worker", "vllm", 0), "the worker tier (presets.vllm.worker)"), + (("brain", "mine", 1), "the brain tier's backup (presets.mine.brain.fallbacks[0])"), + (("brain", "mine", 2), "the brain tier's backup (presets.mine.brain.fallbacks[1])"), + ], +) +def test_cap_setting(args: tuple[str, str, int], expected: str) -> None: + assert cap_setting(*args) == expected diff --git a/core/tstd/config.yaml b/core/tstd/config.yaml index a5ee5cf8..4fb4272b 100644 --- a/core/tstd/config.yaml +++ b/core/tstd/config.yaml @@ -18,6 +18,7 @@ # context_window — max context tokens the model supports # max_output_tokens — max tokens the model may generate; sent as # max_tokens on every turn (TD-5008) +# Thinking counts too: 16k+ for hosted reasoning models # credential — named API key from the credentials catalog # below. Omit on a loopback tier for no key # (TD-1801). An off-box tier with no credential diff --git a/core/tstd/config_help.py b/core/tstd/config_help.py index e5720c1e..35fc6f81 100644 --- a/core/tstd/config_help.py +++ b/core/tstd/config_help.py @@ -120,7 +120,9 @@ TOKENS, ), "TierConfig.max_output_tokens": ( - "Sent as max_tokens, and kept free for the answer out of context_window.", + "Sent as max_tokens, and kept free for the answer out of context_window. A " + "reasoning model's thinking counts against it, so size it for the thinking plus the " + "answer: 16k or more for a hosted reasoning model.", TOKENS, ), "TierConfig.vision": "Accept image attachments on turns routed here, and let fs_read " diff --git a/core/tstd/loop.py b/core/tstd/loop.py index 958ec520..828ceab9 100644 --- a/core/tstd/loop.py +++ b/core/tstd/loop.py @@ -93,6 +93,7 @@ from .keychain import KeychainError from .local_worker import ( effective_tier, + local_worker_tier, mark_cu_tool, preset_fast, session_is_cu_heavy, @@ -105,8 +106,11 @@ from .output_cap import ( LENGTH_FINISH, arguments_complete, + cap_setting, cut_call_reply, + empty_reply_failure, log_output_cap, + raised_output_cap, stub_call, ) from .plan_mode import with_plan_note @@ -156,7 +160,7 @@ ToolDefinition as ProviderToolDefinition, ) from .reply_guard import ReplyGuard, user_blocks -from .reply_retry import GUARDED_CODE, StreamResult, stream_reply +from .reply_retry import GUARDED_CODE, StreamResult, stream_reply, thinking_ate_cap from .router import TierName, TierRouter from .session import QueuedUserMessage, Session, SessionEventLog from .session_goal import ( @@ -932,6 +936,7 @@ async def _stream_and_parse( endpoint_failed, collected_reasoning, resendable, + tier_cfg.max_output_tokens, ) @@ -951,6 +956,9 @@ async def _stream_turn_reply( A stall after output began is withdrawn from the window and the call is made once more under ``provider_retry`` (TD-5159, reply_retry.py). + A reply that spent the output cap thinking is asked once more with a + larger cap, when the window has room for one (TD-5233); every later + attempt of this call keeps the larger cap. """ guard_cfg = tier_cfg.reply_guard @@ -964,15 +972,22 @@ async def _stream_turn_reply( else frozenset() ) last_trip: list[str | None] = [None] + prompt_tokens, _ = estimate_tokens(messages, counter, tool_definitions) + raised = raised_output_cap(tier_cfg.max_output_tokens, tier_cfg.context_window, prompt_tokens) + # max_retries 0 turns this off, as it does the stall retry. + cap_retries = min(1, config.provider_retry.max_retries) if raised is not None else 0 + ate_cap = [False] def attempt(response_id: str) -> Awaitable[StreamResult]: cfg = tier_cfg + if ate_cap[0] and raised is not None: + cfg = cfg.model_copy(update={"max_output_tokens": raised}) guard = None if guard_cfg is not None: guard = ReplyGuard(guard_cfg, allowed_blocks=allowed, tools=bool(tool_definitions)) # After runaway thinking, the retry may answer with thinking off. if last_trip[0] == "thinking_cap" and guard_cfg.retry_extra_body: - cfg = tier_cfg.model_copy( + cfg = cfg.model_copy( update={ "extra_body": merge_extra_body( tier_cfg.extra_body or {}, guard_cfg.retry_extra_body @@ -997,6 +1012,8 @@ async def run() -> StreamResult: last_trip[0] = ( result.error_msg.rsplit(": ", 1)[-1] if result.error_code == GUARDED_CODE else None ) + if thinking_ate_cap(result): + ate_cap[0] = True return result return run() @@ -1006,6 +1023,8 @@ async def run() -> StreamResult: session, config.provider_retry, guard_retries=guard_cfg.retries if guard_cfg is not None else 0, + cap_retries=cap_retries, + raised_cap=raised, ) @@ -2845,11 +2864,29 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: break # exit tool-call loop, wait for next user message if not tool_calls and not (collected_content or "").strip(): + # Stop or cancel that lands as the reply ends is not a + # failed turn (TD-5233), as in the failure branch above. + if session.cancel_requested: + break + if session.stop_turn_requested: + await _finish_stopped_turn(session, tier, turn_start, tracker) + break router.record_failure() - empty_msg = ( - "The model finished without a reply — it spent the " - "turn thinking, or hit the output limit mid-thought. " - "Start a new session if the context is already full." + # The context is the cause only near the compaction budget; + # otherwise a new chat would not help (TD-5233). + prompt_tokens, _ = estimate_tokens(messages, counter, tool_definitions) + sent_cap = reply.max_tokens or tier_cfg.max_output_tokens + # A backup or a remapped worker has its own entry, and that + # is the value that applied. + cap_preset = config.active_preset + if tier == "worker" and cu_heavy and local_worker_tier(config) is not None: + cap_preset = config.computer_use.local_worker_preset.strip() + empty_code, empty_msg = empty_reply_failure( + finish_reason, + setting=cap_setting(tier, cap_preset, backup_chain.position), + cap=tier_cfg.max_output_tokens, + sent=sent_cap, + near_full=prompt_tokens >= budget_threshold(tier_cfg), ) messages.append(ChatMessage(role="assistant", content=empty_msg)) await session.conversation_changed() @@ -2866,7 +2903,7 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: turn_start, tracker, failed=True, - error_code="empty_completion", + error_code=empty_code, ) log.warning( "turn failed: empty completion", @@ -2875,7 +2912,11 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: "session_id": session.id, "tier": tier, "turn": router.turn_count, + "model": model_slug, + "max_tokens": sent_cap, "finish_reason": finish_reason, + "prompt_tokens": prompt_tokens, + "error_code": empty_code, } }, ) @@ -2889,6 +2930,9 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: # A reply stopped at max_output_tokens (TD-5008): cut text is # kept; a cut tool call is never run or replayed as sent. cut_calls: dict[int, dict[str, str | int]] = {} + # A retry after a reply that spent the cap thinking sent a + # larger one (TD-5233); a cut there names that figure. + reply_cap = reply.max_tokens or tier_cfg.max_output_tokens if finish_reason == LENGTH_FINISH: cut_calls = { index: call @@ -2900,7 +2944,7 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: session_id=session.id, tier=tier, model=model_slug, - max_output_tokens=tier_cfg.max_output_tokens, + max_output_tokens=reply_cap, cut_calls=len(cut_calls), ) @@ -2927,8 +2971,7 @@ def counter_for(tier_cfg: TierConfig) -> CalibratedTokenCounter: ) # Answered now, so the model sees why and retries smaller. messages.extend( - cut_call_reply(str(call["id"]), tier_cfg.max_output_tokens) - for call in cut_calls.values() + cut_call_reply(str(call["id"]), reply_cap) for call in cut_calls.values() ) await session.conversation_changed() diff --git a/core/tstd/output_cap.py b/core/tstd/output_cap.py index afceb287..bb73f176 100644 --- a/core/tstd/output_cap.py +++ b/core/tstd/output_cap.py @@ -9,6 +9,12 @@ which some chat templates refuse to render. The call stays in the assistant message with empty arguments, is never dispatched, and is answered with a note that tells the model to send smaller pieces. + +A reasoning model can spend the whole cap thinking: on most providers +thinking tokens count against ``max_tokens``, so the stream ends at +``"length"`` with no answer and no tool call. The turn asks once more with +a larger cap (TD-5233, reply_retry.py), sized here, and when that also ends +with no answer the chat says which setting to raise. """ from __future__ import annotations @@ -22,6 +28,21 @@ log = get_logger("tstd.output_cap") LENGTH_FINISH = "length" +# The assistant_retract reason for the retry after a reply that spent the +# cap thinking, and the turn's error code when it still ends with no answer +# at the cap (TD-5233). +OUTPUT_CAP_THINKING = "output_cap_thinking" +# A reply with no answer that did not end at the cap. +EMPTY_COMPLETION = "empty_completion" +# The retry's cap: this many times the tier's, up to the ceiling (or the +# tier's own cap when that is larger), and never more than the window has +# room for. A raise to less than double is not worth a second call. +THINKING_RETRY_FACTOR = 4 +THINKING_RETRY_CEILING = 32_768 +THINKING_RETRY_MIN_GAIN = 2 +# Kept free in the window on top of the prompt estimate, which can be low. +THINKING_RETRY_MARGIN = 1024 +THINKING_RETRY_MARGIN_FRACTION = 20 # What a cut call is replayed with: valid JSON, and no claim about what # the model meant to send. CUT_CALL_ARGUMENTS = "{}" @@ -93,3 +114,70 @@ def log_output_cap( } }, ) + + +def raised_output_cap(cap: int, context_window: int, prompt_tokens: int) -> int | None: + """The larger ``max_tokens`` to ask again with, or ``None`` (TD-5233). + + *cap* is the tier's ``max_output_tokens``; *prompt_tokens* the + estimate of the prompt the call sends. ``None`` means the window has + no room for at least double the cap, or the cap is already at the + ceiling, so a second call would most likely end the same way. + """ + margin = max(THINKING_RETRY_MARGIN, context_window // THINKING_RETRY_MARGIN_FRACTION) + room = context_window - prompt_tokens - margin + raised = min(cap * THINKING_RETRY_FACTOR, max(THINKING_RETRY_CEILING, cap), room) + return raised if raised >= THINKING_RETRY_MIN_GAIN * cap else None + + +def cap_setting(tier: str, preset: str, fallback: int = 0) -> str: + """Which ``max_output_tokens`` the call used, as the chat names it. + + *fallback* is the backup position (0 is the tier's own model, ``n`` is + ``fallbacks[n - 1]``). The path is the one Settings → Configuration + and config.yaml use, so a backup or a remapped worker names its own + entry rather than the tier's, whose value did not apply. + """ + path = f"presets.{preset}.{tier}" + if fallback: + return f"the {tier} tier's backup ({path}.fallbacks[{fallback - 1}])" + return f"the {tier} tier ({path})" + + +def empty_reply_failure( + finish_reason: str | None, + *, + setting: str, + cap: int, + sent: int, + near_full: bool, +) -> tuple[str, str]: + """``(error_code, chat text)`` for a turn whose reply had no answer. + + *setting* names the config entry (:func:`cap_setting`), *cap* is its + ``max_output_tokens`` and *sent* the ``max_tokens`` + the last call carried, which is larger when the retry ran. The text + is replayed to the model on the next turn, so it stays short and only + states what happened and what to change. *near_full* adds the one case + where a new chat helps: the prompt is at the compaction budget. + """ + if finish_reason == LENGTH_FINISH: + code = OUTPUT_CAP_THINKING + if sent > cap: + text = ( + "No answer: the model used its whole output limit thinking, including one " + f"retry at {sent:,} tokens. Raise max_output_tokens for {setting}, now " + f"{cap:,}, in Settings → Configuration. New chats use the new limit." + ) + else: + text = ( + f"No answer: the model used its whole output limit ({sent:,} tokens) " + f"thinking. Raise max_output_tokens for {setting} in Settings → " + "Configuration. New chats use the new limit." + ) + else: + code = EMPTY_COMPLETION + text = "No answer: the model ended its turn without writing anything. Resend to try again." + if near_full: + text += " The conversation is also close to its context limit, so a new chat may help." + return code, text diff --git a/core/tstd/protocol.py b/core/tstd/protocol.py index aa6e352f..f68c423e 100644 --- a/core/tstd/protocol.py +++ b/core/tstd/protocol.py @@ -2352,7 +2352,11 @@ class AssistantRetract(DaemonEvent): """Withdraw one model response's shown text and thinking (TD-5159). The provider went quiet partway through the reply (``reason`` is the - error code, ``stream_stalled``), and the loop asks again. Every + error code, ``stream_stalled``), and the loop asks again. The same + goes for a reply the endpoint cut off (``stream_interrupted``, + TD-5183), one the reply guard cut (``reply_guarded``, TD-5175), and + one that spent the whole output cap thinking and is asked again with + a larger cap (``output_cap_thinking``, TD-5233). Every ``assistant_delta`` and ``assistant_reasoning`` carrying this ``response_id`` is no longer part of the reply: a client removes it, and the retry streams under a new id. Nothing else on the row goes: @@ -2797,7 +2801,11 @@ class TurnComplete(DaemonEvent): ``failed`` marks a turn that ended on a provider/keychain error rather than a model response; ``error_code`` carries the typed cause (e.g. ``auth_failed``, ``rate_limited``, ``missing_api_key``) so clients can - show tailored copy (TD-1008). + show tailored copy (TD-1008). A reply with no answer fails the turn + too: ``output_cap_thinking`` when it spent the whole output cap + thinking, after one retry with a larger cap where there was room + (TD-5233), and ``empty_completion`` when it ended without writing + anything and did not stop at the cap. """ type: Literal["turn_complete"] = "turn_complete" diff --git a/core/tstd/reply_retry.py b/core/tstd/reply_retry.py index dc23a785..2db77061 100644 --- a/core/tstd/reply_retry.py +++ b/core/tstd/reply_retry.py @@ -13,6 +13,13 @@ partial text never entered the conversation either (TD-5152), so the retry sees exactly what the first attempt saw. +A reply that spent the whole output cap thinking (TD-5233) is not a +fault of the endpoint, so it has its own allowance: it is withdrawn the +same way and asked once more, straight away, with a larger ``max_tokens`` +(output_cap.py sizes it). When the endpoint refuses the larger request +before any output (no credit for it, a bad request), the turn keeps the +first reply and says which setting to raise. + The off-turn calls (the work check, memory distill) never come here and keep TD-5146's rule: a timeout, a stall included, is returned, not resent. """ @@ -27,6 +34,7 @@ from .config import ProviderRetryConfig from .logging import get_logger +from .output_cap import LENGTH_FINISH, OUTPUT_CAP_THINKING from .protocol import AssistantDelta, AssistantRetract, DaemonEvent from .provider import RetryConfig, retry_delay @@ -90,6 +98,9 @@ class StreamResult(NamedTuple): # The provider's flag on a cut-off reply the same endpoint may answer if # asked again (TD-5183). Withdrawn and asked again like a stall. resendable: bool = False + # The max_tokens this call sent: the tier's cap, or the larger one the + # retry after a reply that spent it thinking asked with (TD-5233). + max_tokens: int | None = None def new_response_id() -> str: @@ -111,6 +122,49 @@ def cut_off_retries(stall_allowed: int) -> int: return CUT_OFF_RETRIES if stall_allowed > 0 else 0 +def thinking_ate_cap(result: StreamResult) -> bool: + """True when *result* ended at the output cap with no answer (TD-5233). + + No text and no tool call, yet the stream stopped at ``max_tokens``: + the model spent the cap thinking, which most providers count against + it. + """ + return ( + not result.failed + and result.finish_reason == LENGTH_FINISH + and not result.content.strip() + and not result.tool_calls + ) + + +def _stopping(session: Session) -> bool: + return session.cancel_requested or session.stop_turn_requested + + +def _stopped(result: StreamResult, session: Session) -> StreamResult: + """*result* as a call Stop or cancel ended, with nothing kept.""" + return result._replace( + failed=True, + content="", + tool_calls={}, + error_msg="cancelled" if session.cancel_requested else "stopped", + error_code=None, + raw_detail="", + reasoning_details=None, + ) + + +def _raise_refused(result: StreamResult, session: Session) -> bool: + """The larger-cap request failed before any output, for good. + + A request error (no credit for the larger reservation, a bad request, + an overflow) would fail the turn with an error that hides the real + cause. A failure of the endpoint is not one: the loop's backups take + it. Stop and cancel end the turn as they always do. + """ + return result.failed and not result.began and not result.retryable and not _stopping(session) + + def _withdrawable( result: StreamResult, session: Session, retried: int, allowed: int, guard_retries: int = 0 ) -> bool: @@ -134,6 +188,8 @@ async def stream_reply( retry: ProviderRetryConfig, *, guard_retries: int = 0, + cap_retries: int = 0, + raised_cap: int | None = None, ) -> StreamResult: """Run *attempt* under a fresh response id; withdraw and redo a stall. @@ -142,22 +198,50 @@ async def stream_reply( (``assistant_retract``) and, after ``provider_retry``'s first backoff, run again under a new id. Stop or cancel during the wait ends it with nothing kept: the withdrawn text is not the reply. + + A call that spent the output cap thinking is withdrawn and run again + at once, up to *cap_retries* times; *attempt* sends the larger cap + (*raised_cap*, named in the log) from then on. If the larger request is + refused before any output, the first call's result is returned instead; + a refusal of a later stall or guard retry is returned as itself. """ allowed = stall_retries(retry) retried = 0 + cap_retried = 0 + at_cap: StreamResult | None = None + # Only the call right after the cap retry is the larger request; a + # refusal of a later stall or guard retry is that call's own error. + raise_pending = False while True: response_id = new_response_id() result = await attempt(response_id) + if raise_pending and at_cap is not None and _raise_refused(result, session): + _log_raise_refused(session, result, at_cap) + return at_cap + raise_pending = False + if thinking_ate_cap(result) and cap_retried < cap_retries and not _stopping(session): + await _retract(session, response_id, OUTPUT_CAP_THINKING) + cap_retried += 1 + raise_pending = True + if at_cap is None: + at_cap = result + log.warning( + "reply spent its output cap thinking; asking again with a larger cap", + extra={ + "extra_fields": { + "session_id": session.id, + "max_tokens": result.max_tokens, + "raised_max_tokens": raised_cap, + "attempt": cap_retried, + } + }, + ) + if _stopping(session): + return _stopped(result, session) + continue if not _withdrawable(result, session, retried, allowed, guard_retries): return result - await session.event_log.add( - AssistantRetract( - session_id=session.id, - response_id=response_id, - reason=result.error_code or "stream_stalled", - seq=1, - ) - ) + await _retract(session, response_id, result.error_code or "stream_stalled") retried += 1 delay = retry_delay( retried, @@ -181,15 +265,28 @@ async def stream_reply( ) with contextlib.suppress(TimeoutError): await asyncio.wait_for(session.wait_for_turn_abort(), delay) - if session.cancel_requested or session.stop_turn_requested: - return result._replace( - content="", - tool_calls={}, - error_msg="cancelled" if session.cancel_requested else "stopped", - error_code=None, - raw_detail="", - reasoning_details=None, - ) + if _stopping(session): + return _stopped(result, session) + + +async def _retract(session: Session, response_id: str, reason: str) -> None: + await session.event_log.add( + AssistantRetract(session_id=session.id, response_id=response_id, reason=reason, seq=1) + ) + + +def _log_raise_refused(session: Session, refused: StreamResult, at_cap: StreamResult) -> None: + log.warning( + "the larger output cap was refused; keeping the reply that spent the cap thinking", + extra={ + "extra_fields": { + "session_id": session.id, + "code": refused.error_code, + "max_tokens": at_cap.max_tokens, + "raised_max_tokens": refused.max_tokens, + } + }, + ) def withdrawn_responses(events: Sequence[DaemonEvent]) -> set[str]: diff --git a/docs/architecture.md b/docs/architecture.md index 316041af..fe968a4d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -716,7 +716,7 @@ are stamped by a session's event log, `connection` events fix it at 1, and `ping | `files_restored` | session | Files were put back as they were before a user turn (TD-5125): `restored`, `removed`, `left_alone` (hand-edited files nobody ticked) and `lfs_pointers` (written as a pointer because the object is not stored locally). | | `assistant_delta` | session | A streamed chunk of assistant output. Additive `response_id` names the model response it belongs to (TD-5159); text the daemon writes itself (a failure, a progress note) has none. | | `assistant_reasoning` | session | A streamed chunk of a reasoning model's thinking. Separate from `assistant_delta` because it is not part of the answer: the window folds it behind a disclosure, and it is never replayed to the provider as assistant speech. OpenRouter `reasoning_details` flatten into this event for the UI; the raw array is echoed on the next assistant message after tools, not as speech. Carries the same `response_id` as the deltas (TD-5159). | -| `assistant_retract` | session | Withdraw every `assistant_delta` and `assistant_reasoning` with this `response_id`: the provider went quiet partway through that response (`reason`: `stream_stalled`) and the loop asks again under a new id (TD-5159). Tool calls already run and earlier responses in the turn stay. An older client drops the event and keeps the withdrawn text above the retry. | +| `assistant_retract` | session | Withdraw every `assistant_delta` and `assistant_reasoning` with this `response_id`, and the loop asks again under a new id. `reason` says why: the provider went quiet partway through that response (`stream_stalled`, TD-5159), the endpoint cut it off (`stream_interrupted`, TD-5183), the reply guard cut it (`reply_guarded`, TD-5175), or the response ended at the output cap with only thinking and is asked again with a larger `max_tokens` (`output_cap_thinking`, TD-5233). Before a move to a backup it is the failed call's error code (TD-5120). Tool calls already run and earlier responses in the turn stay. An older client drops the event and keeps the withdrawn text above the retry. | | `tool_call` | session | A tool call about to execute, with its decision class. The class is judged on the same workspace-joined paths dispatch uses, so a relative path shows the class the call runs with; `arguments` stay as the model sent them (TD-5148). | | `tool_result` | session | The outcome of a tool call, with an error code and diff when applicable. `allowed_by` names the read-only shell list entries that let the call run without asking, and is empty when a policy rule, an approval or any other class default decided (TD-5103). `transcript` holds a `delegate` child's steps (task, replies, tool calls and results, notes), for display only: the model gets the capped summary in `output`. A `workflow` call's holds each worker's and checker's steps under a header (TD-5195). It is empty for every other tool (TD-5117). `hooks` lists the after-write hooks (a formatter or linter from `hooks.after_write`) that ran on an `fs_write` or `fs_edit`'s files: each `name`, `status` (`passed`, `failed`, `timed_out`, `error`), `exit_code`, `duration_ms` and whether it `changed` a file. The step timeline shows their time; a failure's output is already in `output`, for the model (TD-5135). | | `shell_output` | session | A streamed chunk of a shell command's stdout or stderr. | diff --git a/docs/configuration.md b/docs/configuration.md index 64b385da..a9ad4c2f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1195,9 +1195,15 @@ withdrawn text above the retried reply; `tst run --output text` has already printed it, so it writes a note to stderr, and `--output json`, scheduled-run receipts and mailed replies leave it out. +A reply that ends at the tier's `max_output_tokens` with only thinking +is withdrawn the same way and asked once more with a larger cap, with no +wait and on its own allowance of one per model call (TD-5233; see "A +reply that spends the whole cap thinking" under "A tier"). +`max_retries: 0` turns that retry off too. + | Key | Type | Default | Effect | |---|---|---|---| -| `max_retries` | int 0–20 | `3` | Retries after the initial attempt. `0` disables retrying; the first failure ends the turn, and a reply that stalls partway is not asked for again (TD-5159). | +| `max_retries` | int 0–20 | `3` | Retries after the initial attempt. `0` disables retrying; the first failure ends the turn, and a reply that stalls partway is not asked for again (TD-5159), nor is a reply that spent the output cap thinking (TD-5233). | | `initial_delay` | float > 0 | `1.0` | Base delay before the first retry, in seconds. Each subsequent wait doubles. | | `max_delay` | float > 0 | `60.0` | Ceiling on any single wait, in seconds. Bounds the worst case when `max_retries` is high. | @@ -1996,7 +2002,7 @@ for a session with the `set_tier` message. | `output_price` | float ≥ 0 | *required* | Dollars per **million** completion tokens. | | `cache_read_price` | float ≥ 0 | *required* | Dollars per **million** prompt tokens served from cache. | | `context_window` | int > 0 | *required* | The compaction budget — see below. | -| `max_output_tokens` | int > 0 | *required* | Sent as `max_tokens` on the tier's requests, and the answer reservation subtracted from `context_window` — see below. | +| `max_output_tokens` | int > 0 | *required* | Sent as `max_tokens` on the tier's requests, and the answer reservation subtracted from `context_window` — see below. A turn's reply that spends all of it thinking is asked once more with a larger cap; see "A reply that spends the whole cap thinking" below (TD-5233). | | `vision` | bool | `false` | When `true`, PNG/JPEG/GIF/WebP attachments are accepted on user turns routed to this tier (TD-4705), and `fs_read` returns an image file, or PDF page images on request, as an image the model sees (TD-5130). Capability is config — never inferred from the slug in code. | | `reasoning_effort` | bool | `false` | When `true`, the picker's effort level (`low`, `medium`, `high`, `xhigh`, `max`) goes out as `reasoning_effort` on this tier's turn requests, and on the requests of delegate and workflow children that run on it (TD-5195). Ultracode is never sent: it is a switch, not a level. Turn it on only for a gateway or provider that supports the field. | | `extra_body` | map | none | Extra JSON fields added to every chat request the tier makes: the turn, the progress check, a delegated worker, the after-write review (validator tier), memory distillation (worker tier), and (on the worker tier) the tool-call classifier, DoD polls and judgment calls that run on the worker. They fill in only fields the request did not set itself. For provider-specific settings such as a llama.cpp server's `chat_template_kwargs` — see §3.6. It cannot set `model`, `messages`, `tools`, `tool_choice`, `stream`, `stream_options`, `max_tokens` or `max_completion_tokens`, and its values must be plain JSON; a load error names the problem. Sent as written: never put a key or token here. | @@ -2043,6 +2049,28 @@ fail the same way. So: and judgment calls that run on the worker tier keep their own 8-token cap; on a reasoning model, turn their thinking off with `verdict_extra_body` (§3.6). +**A reply that spends the whole cap thinking (TD-5233).** When a turn's reply ends at the cap +with only thinking (`finish_reason: length`, no text and no tool call), the daemon withdraws it +(`assistant_retract`, reason `output_cap_thinking`) and sends the same request once more, with +no wait, at a larger `max_tokens`: four times the cap, at most 32768, and no more than the room +left in `context_window` after the prompt and a margin of 5% of the window (at least 1024 +tokens). If that is less than twice the cap there is no retry, so a cap above 16384 is never +raised. A cap of 4096 on a 262144 window with a 28k prompt is asked again at 16384. Later +attempts of the same model call, such as a retry after a stall, keep the larger cap. If the +larger request is refused before any output with an error that is not retried (a 402 because +OpenRouter reserves `max_tokens` against your balance, a 400, a context overflow), the turn +ends as if there had been no retry and the log names the refusal. Stop before the larger request +has written anything keeps nothing; once it is writing, Stop keeps what it wrote, as any Stop +does. A reply that still has no answer fails the turn as `output_cap_thinking`, and the chat +names the entry whose `max_output_tokens` applied (for example `presets.mine.worker`, or +`presets.mine.brain.fallbacks[0]` for a backup) and says to raise it in Settings → Configuration. New chats use the new +limit; an open chat keeps the settings it started with. An empty reply that did not end at the +cap fails as `empty_completion`, and resending is the fix. Either line suggests a new chat only +when the conversation has reached the point where it would be compacted. There is one such +retry per model call, apart from the stall retry, and `provider_retry.max_retries: 0` turns it +off. The progress check, a delegated worker, the after-write review and memory distillation do +not retry this way. + **Seeing and steering compaction (TD-5109).** The title bar shows how full the window is, such as `12k / 64k`, from the daemon's own estimate of the next request (measured as described below). The bar turns amber in the last quarter before compaction and red once the next request @@ -2509,7 +2537,7 @@ verdict_extra_body: A verdict call then sends `chat_template_kwargs: {reasoning_effort: low, enable_thinking: false}`; everything else sends `{reasoning_effort: low}`. Nested maps merge key by key, and `verdict_extra_body` wins where both set the same key. With `reasoning_effort: low` alone, -The GPU server's verdicts came back empty with `finish_reason: length`; with `enable_thinking: false` +the GPU server's verdicts came back empty with `finish_reason: length`; with `enable_thinking: false` the classifier answered in 2 tokens. Check your own server's documentation for its switch. The same reserved keys and the plain-JSON rule apply. When a verdict comes back empty at its cap, the daemon logs one warning per endpoint and model that names `verdict_extra_body`. If @@ -2597,7 +2625,10 @@ Nothing else moves. A rejected key (401), a bad request, an unknown model (404), error or a context overflow would fail the same way anywhere, or needs you, so the turn fails as before, before or partway through a reply. So does a reply whose stream just ends partway, with no finish and no error (`stream_interrupted`), and one that runs past `total_timeout` -(`timeout`): Retry on the message resends it (TD-5152). +(`timeout`): Retry on the message resends it (TD-5152). A reply that spends the whole output +cap thinking is not an endpoint failure either: it is asked again on the same endpoint with a +larger cap (TD-5233, "A tier" above). That larger request moves to a backup only by the rules +above. Each backup gets its own `provider_retry` budget before the next one is asked. When the last backup fails too, the turn fails with that backup's error. diff --git a/docs/tst-desk-backlog.md b/docs/tst-desk-backlog.md index 47ae14f9..bb4ec5d8 100644 --- a/docs/tst-desk-backlog.md +++ b/docs/tst-desk-backlog.md @@ -13042,6 +13042,62 @@ step 1 and the Mac app since TD-5228; on Windows the binary comes from --- +### TD-5233 — A reply that spends the output limit thinking is asked once more, and says so +**Size:** 3 · **Depends on:** TD-5008, TD-5159 +**Status:** In review 2026-11-04. + +A hosted reasoning model on a tier with `max_output_tokens: 4096` spent the +whole cap thinking. The stream ended with `finish_reason: length`, no text +and no tool call. The turn failed as `empty_completion`, and the chat line +said to start a new session if the context was already full, although the +conversation used about a tenth of its window. Nothing asked again, so the +next turns failed the same way. The window had no copy for +`empty_completion`, so a toast also called it an unrecognised error and asked +for a report. OpenRouter, like most providers, counts thinking tokens inside +`max_tokens`. + +**Acceptance criteria:** +- [x] When a turn's model call ends at the output cap with only thinking (`finish_reason: length`, no text, no tool call), the daemon withdraws that response with `assistant_retract` (reason `output_cap_thinking`) and asks once more under a new response id, with no backoff wait +- [x] The retry sends `max_tokens` = min(4 × cap, max(32768, cap), `context_window` − prompt estimate − max(1024, `context_window` / 20)), and only when that is at least twice the cap; otherwise there is no retry (cap 4096, window 262144, prompt 28k: 16384) +- [x] The cap retry is at most one per model call, on its own counter, apart from the stall, cut-off and reply-guard retries; `provider_retry.max_retries: 0` turns it off +- [x] Once a call's cap is raised, its later attempts keep it: a stall retry after the cap retry sends the raised cap, with any `reply_guard` `retry_extra_body` still applied +- [x] A raised request refused before any output with an error that is not retried (a 402 for the balance OpenRouter reserves against `max_tokens`, a 400, a context overflow) leaves the first, empty-at-cap result as the outcome, and a warning names the refusal's code; a retryable failure goes on to the usual retry and backup handling, and a refusal of a later stall or guard retry is that call's own error +- [x] Stop or cancel after the withdrawal and before the larger request writes any text ends the turn with nothing kept; Stop or cancel when an empty reply comes back ends the turn as stopped or cancelled, not as a failure +- [x] A turn that still has no answer at the cap fails as `output_cap_thinking`; the chat line says the model used its whole output limit thinking, names the cap (and the retry's cap when there was one) and the config entry whose value applied (`presets..`, a backup's `fallbacks[n]`, or the computer-use `local_worker_preset`'s worker), and says to raise `max_output_tokens` there in Settings → Configuration, where new chats use the new limit +- [x] An empty reply that did not end at the cap fails as `empty_completion`, with a line saying the model ended its turn without writing anything and to resend +- [x] Either line says a new chat may help only when the prompt estimate is at or past the compaction point; the new-session advice is gone otherwise +- [x] A reply cut at the raised cap logs and names the cap that was sent, not the tier's +- [x] The empty-reply warning in the log names the model, the cap sent, the finish reason, the prompt estimate and the code +- [x] The window has copy for both codes: a banner for `output_cap_thinking` (cleared once a later turn answers, live or replayed) and a toast for `empty_completion`, the Activity title and preview, and the status line; the OS notification uses the copy's title. Neither shows as an unrecognised error +- [x] docs/configuration.md, docs/architecture.md and DECISIONS.md updated +- [x] Tests in core and ui fail on the old code and pass on the new + +Done (2026-11-04): `stream_reply` (reply_retry.py) withdraws a reply that +`thinking_ate_cap` and asks once more on its own counter; `_stream_turn_reply` +sizes the larger cap with `output_cap.raised_output_cap` and keeps it for the +call's later attempts. The loop's empty-reply branch handles Stop first, then +uses `output_cap.empty_reply_failure` and `cap_setting` for the code and the +line. UI: `error-copy.ts`, `timeline.ts`, `first-token-wait.ts`/`retryLine`, +`os-notify.ts`, and `notifications.svelte.ts` drops the banner after a turn +that answers. Tests: `core/tests/test_thinking_cap_retry.py` (45; with the +retry turned off 10 fail, and each later fix has a test that fails without +it), `notifications.test.ts`, `chat-retract.test.ts`, `os-notify.test.ts`, +`chat-pane-retry.test.ts`. Class B entry in DECISIONS.md. Checked live +against OpenRouter on 2026-11-05 with a reasoning model and a tiny cap. +A reply that spent 128 tokens thinking was asked again at 512 and answered. +One that spent 512 and then 2,048 failed with `output_cap_thinking` and the +line naming the setting. + +**Not in this story:** the delegate child, which still ends with "Worker +finished with no summary." for the same failure; the progress check; the +after-write validator review; memory distillation; the Settings model test +probe; sending OpenRouter `reasoning` parameters; parsing +`reasoning_tokens` from usage; a spend-cap check before the retry; machine-aware +failure copy for a session that runs on a paired computer (every turn code +points at this computer's Settings today). + +--- + ### TD-5232 — An element action in full control moves the ring to its display **Size:** 1 · **Depends on:** TD-5226, TD-5227 **Status:** In review 2026-11-03. diff --git a/ui/src/lib/chat-events.ts b/ui/src/lib/chat-events.ts index f287c70f..fd95337b 100644 --- a/ui/src/lib/chat-events.ts +++ b/ui/src/lib/chat-events.ts @@ -339,12 +339,13 @@ export function applyChatEvent(ctx: ChatEventContext, event: DaemonEventUnion): } case "assistant_retract": { if (event.session_id !== state.sessionId) return; - // The provider went quiet partway through this response (TD-5159): - // its text comes off the row, and the daemon is asking again. Until - // the retry's first token the Working line says so. + // The provider went quiet partway through this response (TD-5159), or + // it spent the output limit thinking (TD-5233): its text and thinking + // come off the row, and the daemon is asking again. Until the retry's + // first token the Working line says so, in the reason's words. withdrawResponse(state.messages, event.response_id); state.turnState = "running"; - wait.begin(true); + wait.begin(event.reason); return; } case "turn_complete": { diff --git a/ui/src/lib/chat-pane-retry.test.ts b/ui/src/lib/chat-pane-retry.test.ts new file mode 100644 index 00000000..51989fd4 --- /dev/null +++ b/ui/src/lib/chat-pane-retry.test.ts @@ -0,0 +1,122 @@ +// @vitest-environment jsdom +// +// The chat pane's Working line while the daemon asks again for a reply it +// withdrew, mounted for real (TD-5159, TD-5233). A reply that spent the +// output limit thinking is retried with a larger limit, and the line says +// that instead of calling it interrupted. Only the socket is faked. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { flushSync, mount, unmount } from "svelte"; +import type { ClientMessageUnion, DaemonEventUnion } from "./protocol"; + +const mocks = vi.hoisted(() => ({ + handlers: new Set<(event: DaemonEventUnion) => void>(), +})); + +vi.mock("./connection-status.svelte.js", async (importOriginal) => { + const real = await importOriginal(); + return { + ...real, + onDaemonEvent: (handler: (event: DaemonEventUnion) => void) => { + mocks.handlers.add(handler); + return () => mocks.handlers.delete(handler); + }, + sendToDaemon: (_msg: ClientMessageUnion) => true, + attachToSession: () => {}, + detachFromSession: () => {}, + }; +}); + +import ChatPane from "./components/chat/ChatPane.svelte"; +import { ws } from "./connection-status.svelte.js"; + +let app: ReturnType | null = null; +let seq = 1; + +function emit(event: Record): void { + seq += 1; + const full = { session_id: "s1", seq, ...event } as DaemonEventUnion; + for (const handler of [...mocks.handlers]) handler(full); + flushSync(); +} + +function status(): string { + return document.querySelector(".turn-status")?.textContent ?? ""; +} + +function startTurn(): void { + app = mount(ChatPane, { target: document.body }); + flushSync(); + emit({ + type: "session_list", + sessions: [ + { + session_id: "s1", + workspace_path: "/workspace", + state: "idle", + created_at: "2026-11-04T10:00:00Z", + updated_at: "2026-11-04T10:00:00Z", + event_count: 0, + archived: false, + starred: false, + }, + ], + }); + emit({ type: "user_turn", turn_id: "t1", content: "plan the migration" }); +} + +beforeEach(() => { + mocks.handlers.clear(); + const data = new Map(); + const storage: Storage = { + getItem: (key) => data.get(key) ?? null, + setItem: (key, value) => void data.set(key, value), + removeItem: (key) => void data.delete(key), + clear: () => data.clear(), + key: (index) => [...data.keys()][index] ?? null, + get length() { + return data.size; + }, + }; + Object.defineProperty(window, "localStorage", { configurable: true, value: storage }); + if (typeof globalThis.ResizeObserver === "undefined") { + globalThis.ResizeObserver = class { + observe(): void {} + unobserve(): void {} + disconnect(): void {} + } as unknown as typeof ResizeObserver; + } + ws.state = "connected"; +}); + +afterEach(() => { + if (app !== null) { + unmount(app); + app = null; + } + ws.state = "disconnected"; +}); + +describe("the Working line while a withdrawn reply is asked again", () => { + it("says the model ran out of room to think when thinking used the output limit", () => { + startTurn(); + emit({ type: "assistant_reasoning", delta: "Weighing every option at length", response_id: "r1" }); + emit({ type: "assistant_retract", response_id: "r1", reason: "output_cap_thinking" }); + + expect(status()).toContain("Out of room to think. Asking again with a larger limit…"); + expect(status()).not.toContain("Reply interrupted"); + + // The retry's first thinking token ends the wait; the line goes with it. + emit({ type: "assistant_reasoning", delta: "Shorter.", response_id: "r2" }); + expect(status()).not.toContain("Out of room to think"); + }); + + it("still says the reply was interrupted when the provider went quiet", () => { + startTurn(); + emit({ type: "assistant_delta", delta: "The first step is", response_id: "r1" }); + emit({ type: "assistant_retract", response_id: "r1", reason: "stream_stalled" }); + + expect(status()).toContain("Reply interrupted. Asking again…"); + expect(status()).not.toContain("Out of room to think"); + }); +}); diff --git a/ui/src/lib/chat-retract.test.ts b/ui/src/lib/chat-retract.test.ts index 5f2dcabf..3164747a 100644 --- a/ui/src/lib/chat-retract.test.ts +++ b/ui/src/lib/chat-retract.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { createChatState, createChatStore, type ChatDeps } from "./chat-store"; +import { retryLine } from "./chat-retract"; import type { DaemonEventUnion } from "./protocol"; import { entryTone } from "./entry-view"; import { eventToEntry } from "./timeline"; @@ -21,14 +22,14 @@ const delta = (text: string, responseId?: string): DaemonEventUnion => ev({ type: "assistant_delta", delta: text, ...(responseId ? { response_id: responseId } : {}) }); const thinking = (text: string, responseId: string): DaemonEventUnion => ev({ type: "assistant_reasoning", delta: text, response_id: responseId }); -const retract = (responseId: string): DaemonEventUnion => - ev({ type: "assistant_retract", response_id: responseId, reason: "stream_stalled" }); +const retract = (responseId: string, reason = "stream_stalled"): DaemonEventUnion => + ev({ type: "assistant_retract", response_id: responseId, reason }); const toolCall = (): DaemonEventUnion => ev({ type: "tool_call", tool_call_id: "c1", name: "echo", arguments: { message: "hi" }, decision_class: "A" }); const toolResult = (): DaemonEventUnion => ev({ type: "tool_result", tool_call_id: "c1", status: "success", output: "Echo: hi" }); -const turnComplete = (): DaemonEventUnion => - ev({ type: "turn_complete", tokens: 0, cost: 0, tier: "brain", duration: 1, failed: false, error_code: null }); +const turnComplete = (errorCode: string | null = null): DaemonEventUnion => + ev({ type: "turn_complete", tokens: 0, cost: 0, tier: "brain", duration: 1, failed: errorCode !== null, error_code: errorCode }); function bound() { const deps: ChatDeps = { send: () => true, attach: () => {}, detach: () => {} }; @@ -144,6 +145,7 @@ describe("a reply withdrawn after a stall", () => { const entry = eventToEntry(retract("r1")); expect(entry?.kind).toBe("retry"); expect(entry?.title).toBe("Provider went quiet"); + expect(entry?.preview).toBe("The partial reply was withdrawn; asking again."); expect(entry?.details).toMatchObject({ reason: "stream_stalled", response_id: "r1" }); if (entry) expect(entryTone(entry)).toBe("warning"); }); @@ -176,3 +178,58 @@ describe("a reply withdrawn after a stall", () => { }); }); +// TD-5233: a reasoning model spent the whole output limit thinking and wrote +// nothing. The daemon withdraws that thinking and asks once more with a larger +// limit; if that also ends empty, the turn fails with a line of its own. +describe("a reply that spent the output limit thinking", () => { + const CAP_LINE = + "No answer: the model used its whole output limit thinking, including one retry at 16,384 tokens. Raise max_output_tokens for the brain tier (now 4,096) in Settings → Configuration. New chats use the new limit."; + + it("is a timeline entry that says the retry gets a larger limit", () => { + const entry = eventToEntry(retract("r1", "output_cap_thinking")); + expect(entry?.kind).toBe("retry"); + expect(entry?.title).toBe("Thinking used the output limit"); + expect(entry?.preview).toBe("Asking again with a larger output limit."); + expect(entry?.details).toMatchObject({ reason: "output_cap_thinking", response_id: "r1" }); + }); + + it("says why on the Working line until the retry's first token", () => { + const { state, apply } = bound(); + apply(thinking("Let me weigh every option", "r1"), retract("r1", "output_cap_thinking")); + expect(state.awaitingFirstToken).toBe(true); + expect(state.retryingReply).toBe(true); + expect(state.retryReason).toBe("output_cap_thinking"); + expect(retryLine(state.retryReason)).toBe("Out of room to think. Asking again with a larger limit"); + + apply(thinking("Shorter.", "r2")); + expect(state.awaitingFirstToken).toBe(false); + expect(state.retryingReply).toBe(false); + expect(state.retryReason).toBeNull(); + }); + + it("keeps the interrupted wording for a stall", () => { + const { state, apply } = bound(); + apply(delta("The fix is", "r1"), retract("r1")); + expect(state.retryReason).toBe("stream_stalled"); + expect(retryLine(state.retryReason)).toBe("Reply interrupted. Asking again"); + expect(retryLine(null)).toBe("Reply interrupted. Asking again"); + }); + + it("leaves only the retry's thinking and the failure line when the retry is empty too", () => { + const { state, apply, reply } = bound(); + apply( + thinking("First pass, far too long", "r1"), + retract("r1", "output_cap_thinking"), + thinking("Second pass", "r2"), + // The failure line carries no response id, so nothing withdraws it. + delta(CAP_LINE), + turnComplete("output_cap_thinking"), + ); + + expect(reply().reasoning).toBe("Second pass"); + expect(reply().text).toBe(CAP_LINE); + expect(reply().complete).toBe(true); + expect(state.lastTurnError).toBe("output_cap_thinking"); + expect(state.awaitingFirstToken).toBe(false); + }); +}); diff --git a/ui/src/lib/chat-retract.ts b/ui/src/lib/chat-retract.ts index c869045e..73ca3121 100644 --- a/ui/src/lib/chat-retract.ts +++ b/ui/src/lib/chat-retract.ts @@ -12,6 +12,15 @@ import type { ChatMessage } from "./chat-message"; +/** The Working line while a withdrawn reply is asked again, by the + * retract's reason; the pane adds the ellipsis. A reply that spent the + * output limit thinking is retried with a larger one, and the line says so + * rather than calling it interrupted (TD-5233). */ +export function retryLine(reason: string | null): string { + if (reason === "output_cap_thinking") return "Out of room to think. Asking again with a larger limit"; + return "Reply interrupted. Asking again"; +} + /** Where one model response began on a row: the lengths it found. */ export interface ResponseMark { id: string; diff --git a/ui/src/lib/chat-store.ts b/ui/src/lib/chat-store.ts index aefbddaa..bb430476 100644 --- a/ui/src/lib/chat-store.ts +++ b/ui/src/lib/chat-store.ts @@ -58,6 +58,9 @@ export interface ChatState { /** The current first-token wait is for a reply the daemon withdrew after * the provider went quiet, and is asking for again (TD-5159). */ retryingReply: boolean; + /** The withdrawn reply's `assistant_retract` reason while retrying, else + * null: the Working line words a retry by it (TD-5233). */ + retryReason: string | null; /** Seconds the last turn took, as measured by the daemon on turn_complete. * The UI never times turns itself (AGENTS §6). Cleared on the next send. */ lastTurnDuration: number | null; @@ -158,6 +161,7 @@ export function createChatState(): ChatState { turnStalled: false, awaitingSince: null, retryingReply: false, + retryReason: null, lastTurnDuration: null, lastTurnError: null, draft: "", diff --git a/ui/src/lib/components/chat/ChatPane.svelte b/ui/src/lib/components/chat/ChatPane.svelte index a9e82816..5c98867e 100644 --- a/ui/src/lib/components/chat/ChatPane.svelte +++ b/ui/src/lib/components/chat/ChatPane.svelte @@ -34,6 +34,7 @@ import { settings } from "../../settings.svelte.js"; import { grok } from "../../grok.svelte.js"; import { canSend, formatTurnDuration, showCancel } from "../../chat-store"; + import { retryLine } from "../../chat-retract"; import type { AttachmentDraft } from "../../attachments"; import { parseCompactCommand } from "../../context-gauge"; import { parsePlanCommand, type PlanCommand } from "../../plan-review"; @@ -210,10 +211,11 @@ we know. Cancel stays live below; the first delta recovers this. --> No response yet — the model may be slow or unreachable. {:else if chat.awaitingFirstToken} - + {chat.retryingReply ? "Reply interrupted. Asking again" : workingVerb(waitElapsedMs)}…{chat.retryingReply ? retryLine(chat.retryReason) : workingVerb(waitElapsedMs)}… {#if waitElapsedMs >= 2000} for {formatTurnDuration(waitElapsedMs / 1000)} diff --git a/ui/src/lib/error-copy.ts b/ui/src/lib/error-copy.ts index d20cfe71..b3f359bc 100644 --- a/ui/src/lib/error-copy.ts +++ b/ui/src/lib/error-copy.ts @@ -206,13 +206,33 @@ const TURN_ERROR_COPY: Record = { title: "Provider went quiet", body: "The provider stopped sending for longer than the tier's stream_idle_timeout. Resend to retry. For a slow local model, raise stream_idle_timeout or read_timeout on the tier in config.yaml.", }, + // TD-5233: a reasoning model spent the tier's whole max_output_tokens + // thinking and wrote no answer, even after the daemon's one retry with a + // larger limit. Resending hits the same limit, so a banner. The context + // window is not the cause, so neither entry sends the user to a new chat. + output_cap_thinking: { + severity: "banner", + title: "Model ran out of room to answer", + body: "The model used its whole output limit (max_output_tokens) thinking and never wrote an answer. TST Desk asks once more with a larger limit when there is room. Raise max_output_tokens for this model in Settings → Configuration; chats started after that use it.", + }, + // The reply ended empty before any limit: the next try may well answer. + empty_completion: { + severity: "toast", + title: "No answer", + body: "The model ended its turn without writing anything. Resend to try again; if it keeps happening, try another model.", + }, }; +/** The copy written for this code, or null when the table has none. */ +export function namedTurnFailureCopy(errorCode: string): NoticeSpec | null { + return TURN_ERROR_COPY[errorCode] ?? null; +} + /** Copy for a failed turn's typed error code, or null to stay silent. * `cancelled` ends a session. `stopped` ends only the turn (TD-5003). */ export function turnFailureCopy(errorCode: string | null): NoticeSpec | null { if (errorCode === null || errorCode === "cancelled" || errorCode === "stopped") return null; - const known = TURN_ERROR_COPY[errorCode]; + const known = namedTurnFailureCopy(errorCode); if (known) return known; // Unknown codes still never surface as tracebacks, just as a named code. return { diff --git a/ui/src/lib/first-token-wait.ts b/ui/src/lib/first-token-wait.ts index dda0edb0..927d2c59 100644 --- a/ui/src/lib/first-token-wait.ts +++ b/ui/src/lib/first-token-wait.ts @@ -5,13 +5,14 @@ // promising work was happening. So a wait carries a timer, and past the // threshold the copy stops claiming and says what is actually known. // -// This module owns the timer and the three state fields the shimmer reads, and -// nothing else. It knows no protocol — no session id, no events, no wire. The +// This module owns the timer and the state fields the shimmer reads, and +// nothing else. It knows no protocol — no session id, no events, no wire (a +// retry's reason is held as an opaque string for the pane to word). The // store decides *when* a wait begins and ends, from turn evidence; this decides // what a wait already in progress is worth. /** The slice of ChatState the wait owns. Everything else on the state is the - * store's; nothing here reads or writes past these three fields. */ + * store's; nothing here reads or writes past these fields. */ export interface FirstTokenWaitState { awaitingFirstToken: boolean; turnStalled: boolean; @@ -19,6 +20,9 @@ export interface FirstTokenWaitState { /** This wait is for a reply the daemon withdrew and is asking for again * (TD-5159), not for a send: the Working line says so. */ retryingReply: boolean; + /** Why that reply was withdrawn, as the daemon named it, so the Working + * line can say what the retry changes (TD-5233). Null when not retrying. */ + retryReason: string | null; } /** First-token watchdog (TD-1713): a wait this long with no assistant_delta @@ -29,8 +33,9 @@ export const STALL_TIMEOUT_MS = 25_000; export interface FirstTokenWait { /** Start waiting for the first token, unless a wait is already running. - * `retrying` marks a wait for a withdrawn reply asked again (TD-5159). */ - begin(retrying?: boolean): void; + * `retryReason` marks a wait for a withdrawn reply asked again, and says + * why it was withdrawn (TD-5159, TD-5233). */ + begin(retryReason?: string): void; /** Stop waiting — the token landed, the turn resolved, or the session the * wait belonged to went away. Drops the stall flag with it. */ end(): void; @@ -68,15 +73,19 @@ export function createFirstTokenWait(state: FirstTokenWaitState): FirstTokenWait /** Stamps the elapsed basis and arms the watchdog. A wait already in * progress keeps its original stamp — a replayed "running" must not * restart the user's clock. */ - begin(retrying = false): void { + begin(retryReason?: string): void { if (state.awaitingFirstToken) { - if (retrying) state.retryingReply = true; + if (retryReason !== undefined) { + state.retryingReply = true; + state.retryReason = retryReason; + } return; } state.awaitingFirstToken = true; state.turnStalled = false; state.awaitingSince = Date.now(); - state.retryingReply = retrying; + state.retryingReply = retryReason !== undefined; + state.retryReason = retryReason ?? null; arm(); }, @@ -85,6 +94,7 @@ export function createFirstTokenWait(state: FirstTokenWaitState): FirstTokenWait state.turnStalled = false; state.awaitingSince = null; state.retryingReply = false; + state.retryReason = null; clearStallTimer(); }, diff --git a/ui/src/lib/notifications.svelte.ts b/ui/src/lib/notifications.svelte.ts index ad0c008e..f4f4133c 100644 --- a/ui/src/lib/notifications.svelte.ts +++ b/ui/src/lib/notifications.svelte.ts @@ -126,7 +126,13 @@ export function notifyEvent(event: DaemonEventUnion): void { protocolVersion = event.protocol_version; break; case "turn_complete": - if (!event.failed) break; + if (!event.failed) { + // TD-5233: a turn that answered means "ran out of room to answer" + // no longer holds. This also keeps a replayed history (a chat + // switch, a launch) from leaving it up after a later turn answered. + for (const n of [...banners]) if (n.key === "turn:output_cap_thinking") dismiss(n.id); + break; + } { const spec = turnFailureCopy(event.error_code); if (spec !== null) notify(`turn:${event.error_code}`, spec); diff --git a/ui/src/lib/notifications.test.ts b/ui/src/lib/notifications.test.ts index 8b12dc82..6c68eb9f 100644 --- a/ui/src/lib/notifications.test.ts +++ b/ui/src/lib/notifications.test.ts @@ -528,3 +528,59 @@ describe("checkpoint notices", () => { expect(report).toContain("[REDACTED]"); }); }); + +describe("a turn that ended with no answer (TD-5233)", () => { + // A reasoning model spent the tier's whole output limit thinking. The turn + // used to end as "empty_completion" with no copy, so the user read + // "unrecognised error ... report it" beside a chat line telling them to + // start a new session at 28k of a 256k window. + it("a reply that spent the output limit thinking is a banner naming the setting to raise", () => { + const spec = turnFailureCopy("output_cap_thinking"); + expect(spec?.severity).toBe("banner"); + expect(spec?.title).toBe("Model ran out of room to answer"); + expect(spec?.body).toContain("max_output_tokens"); + expect(spec?.body).toContain("Settings → Configuration"); + expect(spec?.body).not.toContain("unrecognised"); + }); + + it("an empty reply that did not hit the limit is a toast saying to resend", () => { + const spec = turnFailureCopy("empty_completion"); + expect(spec?.severity).toBe("toast"); + expect(spec?.title).toBe("No answer"); + expect(spec?.body).toMatch(/resend/i); + expect(spec?.body).not.toContain("unrecognised"); + }); + + it("neither sends the user to a new session: the context window is not the cause", () => { + for (const code of ["output_cap_thinking", "empty_completion"]) { + const spec = turnFailureCopy(code); + expect(`${spec?.title} ${spec?.body}`).not.toMatch(/new (session|chat)/i); + } + }); + + it("raises one banner per cause, and a repeat updates it in place", () => { + notifyEvent(turnComplete("output_cap_thinking")); + notifyEvent(turnComplete("output_cap_thinking")); + notifyEvent(turnComplete("output_cap_thinking")); + expect(toasts).toHaveLength(0); + expect(banners).toHaveLength(1); + expect(banners[0].key).toBe("turn:output_cap_thinking"); + expect(banners[0].title).toBe("Model ran out of room to answer"); + }); + + it("drops the out-of-room banner once a later turn answers, live or replayed", () => { + notifyEvent(turnComplete("output_cap_thinking")); + notifyEvent(turnComplete(null)); + expect(banners).toHaveLength(0); + // A replay of the same history (chat switch, launch) ends the same way. + notifyEvent(turnComplete("output_cap_thinking")); + notifyEvent(turnComplete(null)); + expect(banners).toHaveLength(0); + }); + + it("keeps other turn banners when a later turn answers", () => { + notifyEvent(turnComplete("insufficient_credits")); + notifyEvent(turnComplete(null)); + expect(banners.map((b) => b.key)).toEqual(["turn:insufficient_credits"]); + }); +}); diff --git a/ui/src/lib/os-notify.test.ts b/ui/src/lib/os-notify.test.ts index 67379d53..4af4113e 100644 --- a/ui/src/lib/os-notify.test.ts +++ b/ui/src/lib/os-notify.test.ts @@ -47,7 +47,7 @@ function approval(): DaemonEventUnion { } as DaemonEventUnion; } -function turn(failed = false): DaemonEventUnion { +function turn(failed = false, errorCode: string | null = failed ? "auth_failed" : null): DaemonEventUnion { return { type: "turn_complete", session_id: "s1", @@ -57,7 +57,7 @@ function turn(failed = false): DaemonEventUnion { tier: "brain", duration: 1, failed, - error_code: failed ? "auth_failed" : null, + error_code: errorCode, } as DaemonEventUnion; } @@ -111,6 +111,20 @@ describe("noticeFor", () => { expect(noticeFor(turn(true))?.title).toBe("Turn failed"); }); + it("says why a turn failed in the window's words, not the raw code (TD-5233)", () => { + expect(noticeFor(turn(true, "output_cap_thinking"))?.body).toBe("Model ran out of room to answer"); + expect(noticeFor(turn(true, "empty_completion"))?.body).toBe("No answer"); + expect(noticeFor(turn(true, "auth_failed"))?.body).toBe("API key rejected"); + }); + + it("keeps a code with no written copy as the code, redacted", () => { + expect(noticeFor(turn(true, "weird_new_code"))?.body).toBe("weird_new_code"); + expect(noticeFor(turn(true, "stopped"))?.body).toBe("stopped"); + expect(noticeFor(turn(true, null))?.body).toBe("The turn did not finish"); + const key = "sk-or-v1-" + "0".repeat(52); // tst-secret-ok + expect(noticeFor(turn(true, `boom ${key}`))?.body).toBe("boom [REDACTED]"); + }); + it("ignores everything else", () => { expect(noticeFor({ type: "assistant_delta", session_id: "s1", seq: 1, delta: "x" } as DaemonEventUnion)).toBeNull(); }); @@ -220,6 +234,15 @@ describe("handleOsEvent", () => { expect(sent[0]?.title).toBe("Turn complete"); }); + it("names the cause of an unfocused failed turn (TD-5233)", async () => { + const { impl, sent } = bridge(); + impl.isPermissionGranted = vi.fn(async () => true); + startOsNotify(impl); + setWindowFocused(false); + await handleOsEvent(turn(true, "output_cap_thinking")); + expect(sent[0]).toMatchObject({ title: "Turn failed", body: "Model ran out of room to answer" }); + }); + it("asks permission only on the first qualifying event", async () => { const { impl } = bridge(); startOsNotify(impl); diff --git a/ui/src/lib/os-notify.ts b/ui/src/lib/os-notify.ts index 6e2018b1..abe60f56 100644 --- a/ui/src/lib/os-notify.ts +++ b/ui/src/lib/os-notify.ts @@ -5,6 +5,7 @@ // does not know about Tauri. import { machineOf } from "./bridged-id"; +import { namedTurnFailureCopy } from "./error-copy"; import type { DaemonEventUnion, JobEntry } from "./protocol"; import { redact } from "./redact"; @@ -78,15 +79,21 @@ export function noticeFor(event: DaemonEventUnion, nameOf?: MachineNamer): OsNot if (event.type === "turn_complete") { return { title: onWhere(event.failed ? "Turn failed" : "Turn complete", event.session_id, nameOf), - body: event.failed - ? redact(event.error_code ?? "The turn did not finish") - : "The agent finished a turn", + body: event.failed ? failedTurnBody(event.error_code) : "The agent finished a turn", kind: "turn", }; } return null; } +/** A failed turn's banner names the cause in the words the window's own + * notice uses (TD-5233). A code with no written copy (and stopped or + * cancelled) stays the code itself, redacted. */ +function failedTurnBody(code: string | null): string { + if (code === null) return "The turn did not finish"; + return namedTurnFailureCopy(code)?.title ?? redact(code); +} + /** A banner is a glance, not a transcript. The daemon caps a run summary * at 2000 characters; that is a paragraph too many for a notification. */ const _BANNER_CHARS = 200; diff --git a/ui/src/lib/timeline.ts b/ui/src/lib/timeline.ts index 8a8a0827..a6e9e877 100644 --- a/ui/src/lib/timeline.ts +++ b/ui/src/lib/timeline.ts @@ -178,14 +178,23 @@ function trimEntry(event: ContextCompacted, trigger: keyof typeof TRIM_PREVIEWS) }; } -/** Map one daemon event to a timeline entry, or null when it isn't shown. */ -/** Why the daemon took a half-written reply back (TD-5159, TD-5175, TD-5183). */ +/** Why the daemon took a half-written reply back (TD-5159, TD-5175, TD-5183, + * TD-5233). */ function retractTitle(reason: string): string { if (reason === "stream_interrupted") return "Reply cut off"; if (reason === "reply_guarded") return "Reply guard cut the reply"; + if (reason === "output_cap_thinking") return "Thinking used the output limit"; return "Provider went quiet"; } +/** What the daemon does next. A reply that spent the output limit thinking + * wrote no answer to withdraw; its retry gets a larger limit (TD-5233). */ +function retractPreview(reason: string): string { + if (reason === "output_cap_thinking") return "Asking again with a larger output limit."; + return "The partial reply was withdrawn; asking again."; +} + +/** Map one daemon event to a timeline entry, or null when it isn't shown. */ export function eventToEntry(event: DaemonEventUnion): TimelineEntry | null { switch (event.type) { case "tool_call": @@ -385,7 +394,7 @@ export function eventToEntry(event: DaemonEventUnion): TimelineEntry | null { kind: "retry", seq: event.seq, title: retractTitle(event.reason), - preview: "The partial reply was withdrawn; asking again.", + preview: retractPreview(event.reason), details: { reason: event.reason, response_id: event.response_id }, }; case "error":