From 53e64bd5a1f78b738392b7055f534b2c5430cda4 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 15:00:08 -0700 Subject: [PATCH 1/5] feat(kotlin): sessions.create opens a watched session Co-authored-by: Cursor --- .../agents/androidassistant/ChatViewModel.kt | 2 +- sdks/kotlin/README.md | 16 +++-- .../agents/android/core/AgentSession.kt | 52 +++++++++++---- .../getstream/agents/android/core/Options.kt | 6 ++ .../getstream/agents/android/core/Sessions.kt | 39 ++++++++--- .../agents/android/core/StreamAgents.kt | 32 ++++----- .../agents/android/core/AgentSessionTest.kt | 65 +++++++++++++++++++ .../getstream/agents/android/core/LiveTest.kt | 4 +- .../agents/android/core/PostureTest.kt | 2 +- .../agents/android/core/StreamAgentsTest.kt | 25 +++---- 10 files changed, 182 insertions(+), 61 deletions(-) diff --git a/examples/voice_agents/android_assistant/app/src/main/kotlin/io/getstream/agents/androidassistant/ChatViewModel.kt b/examples/voice_agents/android_assistant/app/src/main/kotlin/io/getstream/agents/androidassistant/ChatViewModel.kt index 1a3e67fd03..1b960d409f 100644 --- a/examples/voice_agents/android_assistant/app/src/main/kotlin/io/getstream/agents/androidassistant/ChatViewModel.kt +++ b/examples/voice_agents/android_assistant/app/src/main/kotlin/io/getstream/agents/androidassistant/ChatViewModel.kt @@ -347,7 +347,7 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { context = getApplication(), scope = screenScope, start = { - sessionId?.let { Assistant.agents.attach(it, tools) } ?: Assistant.agents.agent(Assistant.AGENT_NAME).chat( + sessionId?.let { Assistant.agents.attach(it, tools) } ?: Assistant.agents.agent(Assistant.AGENT_NAME).sessions.create( SessionOptions(modelOverwrites = ModelOverwrites(llm = Assistant.CHAT_MODEL), tools = tools, returnUrl = Assistant.RETURN_URL), ) }, diff --git a/sdks/kotlin/README.md b/sdks/kotlin/README.md index fc77cd9e30..215f0f7bf1 100644 --- a/sdks/kotlin/README.md +++ b/sdks/kotlin/README.md @@ -33,8 +33,9 @@ dependencies { val agents = StreamAgents(apiKey = "your_api_key") agents.setUser(User(id = "jlahey")) { yourBackend.agentToken() } -// In writing. No call is joined, nothing is transcribed or spoken. -val chat = agents.agent("docs").chat() +// In writing. No call is joined, nothing is transcribed or spoken. It returns with the +// socket open, so the conversation is already being watched. +val chat = agents.agent("docs").sessions.create() chat.responses.create("What are your opening hours?") // chat.conversation is a StateFlow: the question shows at once, the turns grow as the reply // streams in, and its state says what the agent is doing. @@ -95,6 +96,13 @@ streaming in, its reasoning and tool calls, and the approvals waiting on the per `say(text)` has the agent speak words as written, without the model; `say(text, interrupt = true)` stops what it is saying first rather than waiting for it to finish. +`sessions.create` opens a conversation and watches it. `SessionOptions(startVoice = true)` puts +the agent on the session's call as well; `agents.agent("docs").chat(...)` and `.voice(...)` are +the same call with `startVoice` false and true. `SessionOptions(watch = false)` opens one with no +socket and declares no tools: its turns go through `responses.create` and come back from +`responses.items()`, and `say`, `interrupt`, `decide` and `addTools` throw +`AgentsException.Configuration`. The router's row is `.session` either way. + ### Remembering the user `memory` has the agent remember the user `setUser` named: what it learned about them before is @@ -102,7 +110,7 @@ recalled in this session, and what it learns here is kept for their next one. It user's; a device recalls nobody else's, and an `incognito` session remembers nothing. ```kotlin -val chat = agents.agent("docs").chat( +val chat = agents.agent("docs").sessions.create( SessionOptions(memory = SessionOptions.Memory(filter = mapOf("topic" to "billing"))), ) ``` @@ -134,7 +142,7 @@ val lookup = AgentTool( parameters = AgentTool.strings(mapOf("order_id" to "the order number"), required = listOf("order_id")), ) { arguments -> orders.find(arguments["order_id"]?.jsonPrimitive?.content.orEmpty()) } -val chat = agents.agent("docs").chat(lookup) +val chat = agents.agent("docs").sessions.create(SessionOptions(tools = listOf(lookup))) ``` Or let a `@Serializable` class be the arguments and their schema. A property with no default diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/AgentSession.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/AgentSession.kt index 55be08a43e..8323179d7d 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/AgentSession.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/AgentSession.kt @@ -51,6 +51,10 @@ private const val EVENT_BUFFER = 256 * * The session owns a scope, and [close] ends it. It is not tied to a screen: an `AndroidX` * `ViewModel` that holds one and closes it in `onCleared` is the usual home. + * + * One opened with [SessionOptions.watch] false has no socket: its turns go in through + * [responses] and come back out of [Responses.items], and what needs the socket throws + * [AgentsException.Configuration]. */ public class AgentSession internal constructor( /** Where this conversation is held, and who is holding it. */ @@ -58,9 +62,10 @@ public class AgentSession internal constructor( session: Session, tools: List, context: CoroutineContext = Dispatchers.Default, + watched: Boolean = true, ) { private val scope = CoroutineScope(SupervisorJob() + context) - private val socket = SessionSocket( + private val socket = if (!watched) null else SessionSocket( backend, session.id, // Decisions arrive several times a second and are for somebody watching a call, not @@ -153,9 +158,10 @@ public class AgentSession internal constructor( /** * Opens the socket and starts following the conversation. Doing this twice does nothing. * - * The shorthands that return an [AgentSession] have already called it. + * The calls that return an [AgentSession] have already called it. */ public suspend fun start() { + val socket = held() if (!started.compareAndSet(false, true)) return val events = try { socket.open() @@ -179,13 +185,14 @@ public class AgentSession internal constructor( * first, rather than waiting for it to finish. */ public suspend fun say(text: String, interrupt: Boolean = false) { + val socket = held() if (interrupt) socket.send(Command.Interrupt()) socket.send(Command.Say(text)) } /** Abandons the reply in flight, or stops the request named. */ public suspend fun interrupt(requestId: String = "") { - socket.send(Command.Interrupt(requestId)) + held().send(Command.Interrupt(requestId)) } /** @@ -197,6 +204,7 @@ public class AgentSession internal constructor( * sent, the call waits again. */ public suspend fun decide(toolCallId: String, allowed: Boolean, summary: String = "") { + val socket = held() var index = -1 var request: ToolApprovalRequest? = null held.update { waiting -> @@ -228,7 +236,7 @@ public class AgentSession internal constructor( * is left alone. */ public suspend fun declineLogin(toolCallId: String) { - socket.send(Command.ToolApproval(toolCallId, allowed = false)) + held().send(Command.ToolApproval(toolCallId, allowed = false)) } /** Puts the agent on the session's call, `agent:`. See [Sessions.startVoice]. */ @@ -277,6 +285,8 @@ public class AgentSession internal constructor( * their calls are still answered here until adding or removing the name again settles it. */ public suspend fun addTools(tools: List) { + // A conversation nothing is watching has nowhere to be asked for a call. + held() if (tools.isEmpty()) return changing.withLock { // A name left unsettled may or may not be the router's: it is withdrawn first, so @@ -325,6 +335,7 @@ public class AgentSession internal constructor( * the name again settles it. */ public suspend fun removeTool(name: String) { + held() changing.withLock { withdraw(name) } } @@ -333,10 +344,19 @@ public class AgentSession internal constructor( * remembered is kept; [delete] takes it away. */ public suspend fun close() { - try { - socket.send(Command.Close) - } catch (_: AgentsException) { - // Already gone, which is what closing wanted. + if (socket == null) { + try { + Sessions(backend).close(session.id) + } catch (e: AgentsException.Http) { + // Already gone, which is what closing wanted. + if (e.status != 404) throw e + } + } else { + try { + socket.send(Command.Close) + } catch (_: AgentsException) { + // Already gone, which is what closing wanted. + } } end() } @@ -369,8 +389,18 @@ public class AgentSession internal constructor( unsettled.remove(name) } + /** + * The socket, or what to do instead of it: a conversation opened with + * [SessionOptions.watch] false has none, and nothing is watching it to send to. + */ + private fun held(): SessionSocket = socket ?: throw AgentsException.Configuration( + "this conversation is not being watched, so there is nothing to send to: open it " + + "without watch = false, or use responses.create() and read the turn back through " + + "responses.items()", + ) + private suspend fun end() { - socket.close() + socket?.close() scope.cancel() connected.value = false held.value = emptyList() @@ -457,7 +487,7 @@ public class AgentSession internal constructor( Command.ToolResult(call.id, output = e.message.orEmpty(), requestId = call.requestId, turnId = call.turnId) } try { - socket.send(answer) + socket?.send(answer) } catch (_: AgentsException) { // The socket went; there is nobody left to show. } @@ -489,7 +519,7 @@ public class AgentSession internal constructor( Command.ToolResult(call.id, error = e.message ?: e.toString(), requestId = call.requestId, turnId = call.turnId) } try { - socket.send(result) + socket?.send(result) } catch (_: AgentsException) { // The socket went while the tool ran; there is nobody left to tell. } diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Options.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Options.kt index 208a53a6e5..9e21fa635f 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Options.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Options.kt @@ -60,6 +60,12 @@ public data class SessionOptions( val voice: String? = null, /** Functions of yours the agent may call, answered on this device. */ val tools: List = emptyList(), + /** + * Whether to watch the conversation: open its socket, answer [tools] and fold its events + * into [AgentSession.conversation]. False opens it with no socket and declares no tools; + * its turns go through [AgentSession.responses]. + */ + val watch: Boolean = true, /** Cost labels, carried onto every request the session makes. */ val tags: Map = emptyMap(), /** diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt index 7ab351181b..34866bf571 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt @@ -15,9 +15,9 @@ import kotlinx.serialization.json.JsonObject /** * Conversations: opening one, finding the ones there were, and acting on one by id. * - * What comes back are the router's rows rather than live handles, because reading a - * conversation back is not the same as holding one and most of them are over. Holding one is - * [StreamAgents.chat], [StreamAgents.voice] or [StreamAgents.attach]. + * Opening one is holding it. The rest come back as the router's rows rather than live handles, + * because reading a conversation back is not the same as holding one and most of them are + * over; [StreamAgents.attach] holds one by id. * * From [Agent.sessions] every call is about that agent: opening names it, and listing is * narrowed to it. @@ -27,12 +27,19 @@ public class Sessions internal constructor( private val agent: String? = null, ) { /** - * Opens a session without following it, for a caller building its own state layer. + * Opens a conversation and starts watching it, answering [SessionOptions.tools] here. * - * It returns once the router is holding the conversation. Without - * [SessionOptions.startVoice] it is held in writing: nothing is joined, transcribed or spoken. + * It returns with the socket open, so a session that has opened is one that is already + * listening. Without [SessionOptions.startVoice] it is held in writing: nothing is joined, + * transcribed or spoken. With [SessionOptions.watch] false it opens no socket and declares + * no tools, and its turns go through [AgentSession.responses]. */ - public suspend fun create(options: SessionOptions = SessionOptions()): Session { + public suspend fun create(options: SessionOptions = SessionOptions()): AgentSession { + val tools = if (options.watch) options.tools else emptyList() + val duplicate = tools.groupBy { it.name }.entries.firstOrNull { it.value.size > 1 } + if (duplicate != null) { + throw AgentsException.Configuration("${duplicate.key} is declared twice") + } val request = CreateSessionRequest( id = options.id?.ifEmpty { null }, startVoice = if (options.startVoice) true else null, @@ -49,7 +56,7 @@ public class Sessions internal constructor( stt = options.stt, tts = options.tts, voice = options.voice, - tools = options.tools.ifEmpty { null }?.map { it.schema }, + tools = tools.ifEmpty { null }?.map { it.schema }, tags = options.tags.ifEmpty { null }, returnUrl = options.returnUrl?.ifEmpty { null }, memory = options.memory?.schema(backend.user?.id), @@ -61,7 +68,21 @@ public class Sessions internal constructor( request, SessionSchema.serializer(), ) - return Session.of(created) + val session = AgentSession(backend, Session.of(created), tools, watched = options.watch) + if (!options.watch) return session + try { + session.start() + } catch (e: AgentsException) { + // The session is live in the router even though nothing here can watch it, so it is + // stopped rather than left holding a call nobody is listening to. + try { + close(created.id) + } catch (_: AgentsException) { + // What the caller needs to hear is why it could not be watched. + } + throw e + } + return session } /** diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/StreamAgents.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/StreamAgents.kt index 25b2ca50a2..a8b9c13f72 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/StreamAgents.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/StreamAgents.kt @@ -8,7 +8,7 @@ import okhttp3.OkHttpClient * Two lines get a conversation going: * * val agents = StreamAgents(url = "https://your-router", customerId = "acme") - * val chat = agents.agent("docs").chat() + * val chat = agents.agent("docs").sessions.create() * * Opening a conversation, reading back its turns, going back to one of them, branching off and * ending it is the whole of what is here, because it is the whole of what the router lets a @@ -50,22 +50,23 @@ public class StreamAgents(public val backend: Backend) : AutoCloseable { /** * Holds a conversation in writing: no call is joined, nothing is transcribed or spoken. + * [Sessions.create] without [SessionOptions.startVoice]. * * The replies still come through the model with the same instructions, skills and knowledge - * a call would have had, and stream into [AgentSession.conversation]. It returns with the - * socket open, so the tools are already being answered. + * a call would have had, and stream into [AgentSession.conversation]. */ public suspend fun chat(options: SessionOptions = SessionOptions()): AgentSession = - follow(sessions.create(checked(options.copy(startVoice = false))), options.tools) + sessions.create(options.copy(startVoice = false)) /** * Puts an agent on the session's own call, `agent:`, and follows it. + * [Sessions.create] with [SessionOptions.startVoice]. * * The agent joins as soon as this returns. Joining the same call from this device is what * [VoiceSession] is for; this only starts the agent and gives you the state. */ public suspend fun voice(options: SessionOptions = SessionOptions()): AgentSession = - follow(sessions.create(checked(options.copy(startVoice = true))), options.tools) + sessions.create(options.copy(startVoice = true)) /** * Follows a session this caller already has open, without creating one. @@ -75,7 +76,7 @@ public class StreamAgents(public val backend: Backend) : AutoCloseable { * reading a conversation. */ public suspend fun attach(sessionId: String, tools: List = emptyList()): AgentSession = - follow(sessions.get(sessionId), tools) + AgentSession(backend, sessions.get(sessionId), tools).also { it.start() } /** * Gets or creates a guest, so somebody can talk to an agent before they sign up. @@ -110,17 +111,6 @@ public class StreamAgents(public val backend: Backend) : AutoCloseable { override fun close() { backend.close() } - - private suspend fun follow(session: Session, tools: List): AgentSession = - AgentSession(backend, session, tools).also { it.start() } - - private fun checked(options: SessionOptions): SessionOptions { - val duplicate = options.tools.groupBy { it.name }.entries.firstOrNull { it.value.size > 1 } - if (duplicate != null) { - throw AgentsException.Configuration("${duplicate.key} is declared twice") - } - return options - } } /** @@ -133,15 +123,15 @@ public class Agent internal constructor(private val agents: StreamAgents, public /** This agent's conversations: opening one names it, and listing is narrowed to it. */ public val sessions: Sessions = Sessions(agents.backend, name) - /** Holds a conversation with this agent in writing. See [StreamAgents.chat]. */ + /** Holds a conversation with this agent in writing. [Sessions.create] without voice. */ public suspend fun chat(options: SessionOptions = SessionOptions()): AgentSession = - agents.chat(options.copy(agent = options.agent ?: name)) + sessions.create(options.copy(startVoice = false)) /** Holds a conversation with this agent in writing, answering these tools here. */ public suspend fun chat(vararg tools: AgentTool): AgentSession = chat(SessionOptions(tools = tools.toList())) - /** Puts this agent on the session's own call and follows it. See [StreamAgents.voice]. */ + /** Puts this agent on the session's own call and follows it. [Sessions.create] with voice. */ public suspend fun voice(options: SessionOptions = SessionOptions()): AgentSession = - agents.voice(options.copy(agent = options.agent ?: name)) + sessions.create(options.copy(startVoice = true)) } diff --git a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/AgentSessionTest.kt b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/AgentSessionTest.kt index 190eee0f7f..a2d70e5f59 100644 --- a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/AgentSessionTest.kt +++ b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/AgentSessionTest.kt @@ -177,6 +177,57 @@ class AgentSessionTest { chat.close() } + @Test + fun `opening a session watches it and answers its tools`() = runBlocking { + val lookup = AgentTool("lookup_order", "Look up an order.") { "Order A-1042: two wool throws." } + + val chat = agents.agent("docs").sessions.create(SessionOptions(tools = listOf(lookup))) + + assertTrue(chat.isConnected.value) + assertEquals("/v1/agents/sessions/s1/events", router.handshakes.single().path) + router.script.send(Script.Send("""{"type":"tool_call","id":"c1","name":"lookup_order","arguments":"","request_id":"r1","turn_id":"t1"}""")) + val answer = Json.parseToJsonElement(router.nextSent()).jsonObject + assertEquals(JsonPrimitive("c1"), answer["tool_call_id"]) + assertEquals(JsonPrimitive("Order A-1042: two wool throws."), answer["output"]) + chat.close() + } + + @Test + fun `an agent's chat with tools still answers them`() = runBlocking { + val lookup = AgentTool("lookup_order", "Look up an order.") { "Order A-1042: two wool throws." } + + val chat = agents.agent("docs").chat(lookup) + + router.script.send(Script.Send("""{"type":"tool_call","id":"c1","name":"lookup_order","arguments":"","request_id":"r1","turn_id":"t1"}""")) + val answer = Json.parseToJsonElement(router.nextSent()).jsonObject + assertEquals(JsonPrimitive("Order A-1042: two wool throws."), answer["output"]) + chat.close() + } + + @Test + fun `a session opened unwatched has no socket, and its turns go through responses`() = runBlocking { + val lookup = AgentTool("lookup_order", "Look up an order.") { "" } + + val chat = agents.agent("docs").sessions.create(SessionOptions(tools = listOf(lookup), watch = false)) + + val created = router.requests.single { it.path == "/v1/agents/sessions" } + assertFalse("tools" in Json.parseToJsonElement(created.body).jsonObject.keys) + assertTrue(router.handshakes.isEmpty()) + assertFalse(chat.isConnected.value) + val refused = assertFailsWith { chat.say("hello") } + assertTrue("watch = false" in refused.message.orEmpty(), refused.message) + assertTrue("responses.create" in refused.message.orEmpty(), refused.message) + assertFailsWith { chat.addTools(listOf(lookup)) } + + val asked = chat.responses.create("What are your hours?") + assertEquals("r1", asked.id) + assertEquals("POST /v1/agents/sessions/s1/responses", router.requests.last().let { "${it.method} ${it.path}" }) + + chat.close() + assertEquals("POST /v1/agents/sessions/s1/stop", router.requests.last().let { "${it.method} ${it.path}" }) + assertTrue(router.handshakes.isEmpty()) + } + @Test fun `a tool that throws tells the model it did not work, since it is mid-sentence waiting`() = runBlocking { val broken = AgentTool("lookup_order", "Look up an order.") { error("the orders database is down") } @@ -793,6 +844,20 @@ class AgentSessionTest { assertEquals("req-3", refused.requestId) } + @Test + fun `a session whose socket is refused is stopped, and the refusal is what the caller hears`() = runBlocking { + router.refuseHandshake = Reply(403, envelope("permission", "forbidden", "not your session")) + + val refused = assertFailsWith { agents.agent("docs").sessions.create() } + + assertEquals(403, refused.status) + assertEquals("forbidden", refused.code) + assertEquals( + listOf("POST /v1/agents/sessions", "POST /v1/agents/sessions/s1/stop"), + router.requests.map { "${it.method} ${it.path}" }, + ) + } + @Test fun `a socket refused for its credential is a status too, not a transport failure`() = runBlocking { router.refuseHandshake = Reply(401, envelope("authentication", "unauthenticated", "token expired")) diff --git a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/LiveTest.kt b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/LiveTest.kt index 2f3dfef807..cee1fc8b1c 100644 --- a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/LiveTest.kt +++ b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/LiveTest.kt @@ -121,7 +121,7 @@ class LiveTest { @Test fun `a titled conversation is found by its title`() = runBlocking { val title = "kotlin search ${System.nanoTime()}" - val session = agents.sessions.create(options.copy(title = title)) + val session = agents.sessions.create(options.copy(title = title, watch = false)) val found = agents.sessions.search(title) @@ -149,7 +149,7 @@ class LiveTest { assertTrue(guest.id.isNotEmpty() && guest.token.isNotEmpty()) agents.setUser(guest) - val session = agents.sessions.create(options) + val session = agents.sessions.create(options.copy(watch = false)) assertTrue(agents.sessions.query().items.any { it.id == session.id }) agents.sessions.close(session.id) diff --git a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/PostureTest.kt b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/PostureTest.kt index 1a1de4cc78..e892a20a21 100644 --- a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/PostureTest.kt +++ b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/PostureTest.kt @@ -94,7 +94,7 @@ class PostureTest { } val sessions = agents.sessions - sessions.create() + sessions.create(SessionOptions(watch = false)) sessions.query() sessions.search("billing") sessions.get("s1") diff --git a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/StreamAgentsTest.kt b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/StreamAgentsTest.kt index b1b2a297a6..0d05b9194d 100644 --- a/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/StreamAgentsTest.kt +++ b/sdks/kotlin/core/src/test/kotlin/io/getstream/agents/android/core/StreamAgentsTest.kt @@ -175,8 +175,9 @@ class VisionAgentsTest { modelOverwrites = ModelOverwrites(thinking = ModelOverwrites.Thinking.High, temperature = 0.0), greeting = Greeting("Hello.", Greeting.Mode.Variation), returnUrl = "com.example.app:/connected", + watch = false, ), - ) + ).session val body = Json.parseToJsonElement(router.requests.single().body).jsonObject assertEquals( @@ -203,9 +204,9 @@ class VisionAgentsTest { router.answer = { Reply(201, sessionJson()) } deviceZoneId = { "Asia/Kolkata" } - agents.sessions.create() - agents.agent("docs").sessions.create(SessionOptions(title = "Billing")) - agents.sessions.create(SessionOptions(timezone = "Europe/Skopje")) + agents.sessions.create(SessionOptions(watch = false)) + agents.agent("docs").sessions.create(SessionOptions(title = "Billing", watch = false)) + agents.sessions.create(SessionOptions(timezone = "Europe/Skopje", watch = false)) val (device, agent, named) = router.requests.map { Json.parseToJsonElement(it.body).jsonObject } assertEquals("""{"timezone":"Asia/Kolkata"}""", device.toString()) @@ -218,8 +219,8 @@ class VisionAgentsTest { router.answer = { Reply(201, sessionJson()) } deviceZoneId = { "Asia/Kolkata" } - agents.sessions.create(SessionOptions(timezone = null)) - agents.sessions.create(SessionOptions(timezone = "")) + agents.sessions.create(SessionOptions(timezone = null, watch = false)) + agents.sessions.create(SessionOptions(timezone = "", watch = false)) assertEquals(listOf("{}", "{}"), router.requests.map { it.body }) } @@ -229,9 +230,9 @@ class VisionAgentsTest { router.answer = { Reply(201, sessionJson()) } deviceZoneId = { "GMT+05:30" } - agents.sessions.create() + agents.sessions.create(SessionOptions(watch = false)) // The router is the one that knows which zones it knows, so what the app names is its to refuse. - agents.sessions.create(SessionOptions(timezone = "GMT+05:30")) + agents.sessions.create(SessionOptions(timezone = "GMT+05:30", watch = false)) val (device, named) = router.requests.map { it.body } assertEquals("{}", device) @@ -255,9 +256,9 @@ class VisionAgentsTest { agents.setUser(User("jlahey"), "unused") agents.sessions.create( - SessionOptions(memory = SessionOptions.Memory(filter = mapOf("topic" to "billing"), appId = "web")), + SessionOptions(memory = SessionOptions.Memory(filter = mapOf("topic" to "billing"), appId = "web"), watch = false), ) - agents.sessions.create(SessionOptions(memory = SessionOptions.Memory(), incognito = true)) + agents.sessions.create(SessionOptions(memory = SessionOptions.Memory(), incognito = true, watch = false)) val (narrowed, incognito) = router.requests.map { Json.parseToJsonElement(it.body).jsonObject } assertEquals("""{"app_id":"web","filter":{"topic":"billing"},"user_id":"jlahey"}""", narrowed["memory"].toString()) @@ -271,7 +272,7 @@ class VisionAgentsTest { router.answer = { Reply(201, sessionJson()) } agents.setUser(User("jlahey"), "unused") - agents.sessions.create() + agents.sessions.create(SessionOptions(watch = false)) assertEquals("{}", router.requests.single().body) } @@ -287,7 +288,7 @@ class VisionAgentsTest { fun `a voice session asks to start voice and is on its own call`() = runTest { router.answer = { Reply(201, sessionJson(callId = "s1")) } - val session = agents.sessions.create(SessionOptions(configId = "cfg-1", startVoice = true)) + val session = agents.sessions.create(SessionOptions(configId = "cfg-1", startVoice = true, watch = false)).session val body = Json.parseToJsonElement(router.requests.single().body).jsonObject assertEquals(setOf("config_id", "start_voice"), body.keys) From 5f57bb69ffd96058db91d8aae521aebf0d2cf3f0 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 15:00:08 -0700 Subject: [PATCH 2/5] feat(dart): sessions.create opens a watched session Co-authored-by: Cursor --- sdks/dart/stream_agents/CHANGELOG.md | 11 +++ sdks/dart/stream_agents/README.md | 13 ++- .../lib/src/core/agent_session.dart | 28 +++++- .../stream_agents/lib/src/core/convert.dart | 5 +- .../stream_agents/lib/src/core/models.dart | 9 ++ .../stream_agents/lib/src/core/sessions.dart | 69 ++++++++++---- .../lib/src/core/stream_agents.dart | 48 +++------- .../stream_agents/test/core/client_test.dart | 93 +++++++++++++++++-- .../stream_agents/test/core/live_test.dart | 2 +- .../stream_agents/test/core/posture_test.dart | 2 +- 10 files changed, 213 insertions(+), 67 deletions(-) diff --git a/sdks/dart/stream_agents/CHANGELOG.md b/sdks/dart/stream_agents/CHANGELOG.md index f3b9d3b4d2..ebcebd985d 100644 --- a/sdks/dart/stream_agents/CHANGELOG.md +++ b/sdks/dart/stream_agents/CHANGELOG.md @@ -1,7 +1,18 @@ ## Upcoming +🔄 Changed + +- `sessions.create` opens the session and follows it, as `chat` and `voice` do: it returns an + `AgentSession` once the socket is open, and answers the device's tools. It returned a `Session` + row before; read that as `session.session`. `chat` and `voice` are now `sessions.create` with + `startVoice` false and true. + ✅ Added +- `SessionOptions.watch` - false opens a session with no socket and no tools declared; turns go + through `responses.create`, and `say`, `interrupt`, `events` and `addTools` throw a + `StateError` saying so. + - `StreamAgents.scheduledTasks` - list, change, pause, resume and delete the user's scheduled tasks. - `session.interrupt(requestId:)` - stops only the reply answering that request. - `session.say(text, interrupt: true)` - stops what the agent is saying first. diff --git a/sdks/dart/stream_agents/README.md b/sdks/dart/stream_agents/README.md index 13f527a027..4794151401 100644 --- a/sdks/dart/stream_agents/README.md +++ b/sdks/dart/stream_agents/README.md @@ -83,10 +83,19 @@ final agents = StreamAgents( ### Start a conversation ```dart -final session = await agents.agent('support').chat(); +final session = await agents.agent('support').sessions.create(); await session.responses.create('What are your opening hours?'); ``` +`sessions.create` opens the session and follows it: it returns once the socket is open, so the +device's tools are answered from the first turn. `SessionOptions(startVoice: true)` puts the agent +on the session's call instead of holding it in writing. `chat()` and `voice()` are the same call +with `startVoice` false and true. + +`SessionOptions(watch: false)` opens it without a socket, for a caller reading and writing turns +over HTTP alone: no tool is declared, turns go in through `responses.create` and are read back +from `responses.items`, and `say`, `interrupt`, `events` and `addTools` throw a `StateError`. + `session.conversation` is a `LiveValue`: the current transcript and what the agent is doing, plus a stream of every change. A widget renders it with a `StreamBuilder`: @@ -111,7 +120,7 @@ final lookup = AgentTool( run: (arguments) async => orders.find('${arguments['order_id']}'), ); -final session = await agents.agent('support').chat(SessionOptions(tools: [lookup])); +final session = await agents.agent('support').sessions.create(SessionOptions(tools: [lookup])); ``` The agent only sees what the tool returns. diff --git a/sdks/dart/stream_agents/lib/src/core/agent_session.dart b/sdks/dart/stream_agents/lib/src/core/agent_session.dart index b7d0a4073e..c1cca06d0f 100644 --- a/sdks/dart/stream_agents/lib/src/core/agent_session.dart +++ b/sdks/dart/stream_agents/lib/src/core/agent_session.dart @@ -73,6 +73,7 @@ final class AgentSession { required this._sessions, List tools = const [], this.interim = false, + this._watch = true, }) : _session = session, _tools = {for (final tool in tools.reversed) tool.name: tool}; @@ -105,6 +106,10 @@ final class AgentSession { final Backend _backend; final Sessions _sessions; + /// Whether the conversation is followed over a socket. False is `SessionOptions.watch` + /// false: read and written over HTTP, with nothing here to send to. + final bool _watch; + /// The tools whose calls are answered here, by name. final Map _tools; @@ -160,8 +165,12 @@ final class AgentSession { /// Opens the socket and starts following the conversation. Calling it again, even while it /// is opening, opens nothing more, and after [close] it opens nothing at all. /// - /// Throws, and reports it on [connection], when the socket cannot be opened. - Future start() => _starting ??= _start(); + /// Throws, and reports it on [connection], when the socket cannot be opened, and throws a + /// [StateError] for a session opened with `watch: false`. + Future start() { + _watched(); + return _starting ??= _start(); + } Future _start() async { final Uri uri; @@ -207,6 +216,7 @@ final class AgentSession { /// Each call is a stream of its own, so two listeners each see every event. A listener /// more than 256 events behind is finished rather than buffered without bound. Stream events() { + _watched(); final subscriber = _Subscriber(); if (_connection.value is Disconnected) { subscriber.controller.close(); @@ -300,6 +310,8 @@ final class AgentSession { /// under each. Only a session this device opened takes tools from it, and a name the agent /// has of its own, such as a connector's tool, is refused. Future addTools(List tools) async { + // A conversation nothing is watching has nowhere to be asked for a call. + _watched(); if (tools.isEmpty) { return; } @@ -398,6 +410,7 @@ final class AgentSession { sessions: _sessions, tools: _tools.values.toList(), interim: interim, + watch: _watch, ).._unsettled.addAll(_unsettled), ); @@ -502,7 +515,18 @@ final class AgentSession { return () => _conversation.value = _conversation.value.withoutTurn(id); } + void _watched() { + if (!_watch) { + throw StateError( + 'this conversation is not being watched, so there is nothing to send to: open it ' + 'without watch: false, or use responses.create() and read the turn back through ' + 'responses.items()', + ); + } + } + SessionSocket _held() { + _watched(); final socket = _socket; if (socket == null) { throw StateError('start() the session before talking to it'); diff --git a/sdks/dart/stream_agents/lib/src/core/convert.dart b/sdks/dart/stream_agents/lib/src/core/convert.dart index a0a1c55949..af7fa383fa 100644 --- a/sdks/dart/stream_agents/lib/src/core/convert.dart +++ b/sdks/dart/stream_agents/lib/src/core/convert.dart @@ -100,7 +100,10 @@ api.CreateSessionRequest createRequestOf( custom: options.custom, incognito: options.incognito, modelOverwrites: overwritesOf(options.modelOverwrites), - tools: options.tools.isEmpty ? null : [for (final tool in options.tools) sessionToolOf(tool)], + // Unwatched, nothing here answers a call, so no tool is declared. + tools: !options.watch || options.tools.isEmpty + ? null + : [for (final tool in options.tools) sessionToolOf(tool)], tags: options.tags.isEmpty ? null : options.tags, returnUrl: _set(options.returnUrl), // The router remembers only for a user id, and a device only ever means its own user. diff --git a/sdks/dart/stream_agents/lib/src/core/models.dart b/sdks/dart/stream_agents/lib/src/core/models.dart index 51ebadff38..e725be4a69 100644 --- a/sdks/dart/stream_agents/lib/src/core/models.dart +++ b/sdks/dart/stream_agents/lib/src/core/models.dart @@ -410,6 +410,7 @@ final class SessionOptions { this.returnUrl, this.memory, this.timezone, + this.watch = true, }); /// An id to hold the session by, for a caller that wants to know it before the session @@ -476,6 +477,14 @@ final class SessionOptions { /// and it is remembered as the user's for the scheduled tasks that follow. Null sends none, /// and a zone the router does not know is refused. final String? timezone; + + /// Whether to follow the conversation over its socket, which is what answers [tools] and + /// reports what the agent is doing word by word. + /// + /// False opens it without a socket: [tools] are not declared, nothing here answers a call, + /// and `say`, `interrupt`, `decide`, `events` and `addTools` throw a [StateError]. Turns go + /// in through `responses.create` and are read back from `responses.items`. + final bool watch; } /// What a session remembers about the user [Backend.userId] names. It is always theirs: a diff --git a/sdks/dart/stream_agents/lib/src/core/sessions.dart b/sdks/dart/stream_agents/lib/src/core/sessions.dart index deff2e76c5..465f0267f0 100644 --- a/sdks/dart/stream_agents/lib/src/core/sessions.dart +++ b/sdks/dart/stream_agents/lib/src/core/sessions.dart @@ -1,6 +1,9 @@ import 'dart:math'; +import 'agent_session.dart'; +import 'backend.dart'; import 'convert.dart'; +import 'errors.dart'; import 'generated/api.dart' as api; import 'models.dart'; import 'tools.dart'; @@ -10,26 +13,29 @@ const _itemPage = 200; /// Conversations: the ones being held and the ones that were. /// -/// What comes back are rows rather than live handles, because reading a conversation back is -/// not holding one and most of these are over. `StreamAgents.attach` follows one that is -/// still running. +/// [create] opens one and follows it. What is read back are rows rather than live handles, +/// because reading a conversation back is not holding one and most of these are over. +/// `StreamAgents.attach` follows one that is still running. final class Sessions { - Sessions(this._operations, {this.agent, this._userId = ''}); + Sessions(this._operations, this._backend, {this.agent}); final api.Operations _operations; - /// Who is asking, which is whose memories a session reads. - final String _userId; + /// Where sessions opened here are followed, and who is asking, which is whose memories a + /// session reads. + final Backend _backend; /// The agent name every call here is about, or null for all of this caller's. final String? agent; - /// Opens a conversation without following it, for a caller building its own state layer. + /// Opens a conversation and follows it, answering [SessionOptions.tools] on this device. /// /// Held in writing unless [SessionOptions.startVoice] is set. It returns once the backend is - /// holding the conversation, so a session that comes back is one already listening. - Future create([SessionOptions options = const SessionOptions()]) => - _create(options, options.startVoice); + /// holding the conversation and the socket is open, so a session that comes back is one + /// already listening. With [SessionOptions.watch] false no socket is opened and no tool + /// declared; its turns go through [AgentSession.responses]. + Future create([SessionOptions options = const SessionOptions()]) => + _open(options, voice: options.startVoice); /// A page of this caller's conversations, the ones that ended included. /// @@ -155,9 +161,36 @@ final class Sessions { /// A conversation's turns, whether or not this process is holding it. Responses responses(String id) => Responses._(_operations, id); - Future _create(SessionOptions options, bool startVoice) async { - final request = createRequestOf(options, startVoice: startVoice, agent: agent, userId: _userId); - return sessionOf(await _operations.createSession(body: request)); + Future _open(SessionOptions options, {required bool voice}) async { + final request = createRequestOf( + options, + startVoice: voice, + agent: agent, + userId: _backend.userId, + ); + final session = AgentSession( + sessionOf(await _operations.createSession(body: request)), + backend: _backend, + sessions: this, + tools: options.watch ? options.tools : const [], + watch: options.watch, + ); + if (!options.watch) { + return session; + } + try { + await session.start(); + } on AgentsException { + // The agent is in the conversation whether or not anybody here can follow it, so it is + // stopped rather than left holding a call nobody is listening to. + try { + await stop(session.id); + } on AgentsException { + // Reporting why the socket did not open matters more than this. + } + rethrow; + } + return session; } Future> _query( @@ -368,9 +401,13 @@ final class Responses { } } -/// Opens a conversation on its call or in writing, whatever [options] say. -Future openSession(Sessions sessions, SessionOptions options, {required bool voice}) => - sessions._create(options, voice); +/// Opens a conversation on its call or in writing, whatever [options] say, as +/// [Sessions.create] does. +Future openSession( + Sessions sessions, + SessionOptions options, { + required bool voice, +}) => sessions._open(options, voice: voice); /// Gives a running session more of its caller's tools, declared as when it opened. Future addSessionTools(Sessions sessions, String id, List tools) => diff --git a/sdks/dart/stream_agents/lib/src/core/stream_agents.dart b/sdks/dart/stream_agents/lib/src/core/stream_agents.dart index 5085d68836..e5e0026f9c 100644 --- a/sdks/dart/stream_agents/lib/src/core/stream_agents.dart +++ b/sdks/dart/stream_agents/lib/src/core/stream_agents.dart @@ -18,7 +18,7 @@ import 'wire.dart'; /// Two lines get a conversation going: /// /// final agents = StreamAgents(url: Uri.parse('https://your-router'), customerId: 'acme'); -/// final chat = await agents.agent('docs').chat(); +/// final session = await agents.agent('docs').sessions.create(); /// /// Opening a conversation, following it, reading back its turns, going back to one of them, /// branching off and ending it, a guest to do it as, and looking something up are the whole @@ -50,7 +50,7 @@ final class StreamAgents { : _client = httpClient ?? http.Client(), _ownsClient = httpClient == null { _operations = api.Operations(HttpWire(backend, _client)); - sessions = Sessions(_operations, userId: backend.userId); + sessions = Sessions(_operations, backend); scheduledTasks = ScheduledTasks(_operations); } @@ -78,19 +78,21 @@ final class StreamAgents { withUser(guest.id, token: backend.apiKey.isEmpty ? null : () async => guest.token); /// Holds a conversation in writing: no call is joined, nothing is transcribed or spoken. + /// [Sessions.create] with [SessionOptions.startVoice] false. /// /// The replies still come from the model with the same instructions, skills and knowledge a /// call would have, and arrive as deltas on the socket, which is already open when this /// returns. Future chat([SessionOptions options = const SessionOptions()]) => - _follow(sessions, options, voice: false); + openSession(sessions, options, voice: false); /// Puts an agent on the session's own call, `agent:`, and follows it. + /// [Sessions.create] with [SessionOptions.startVoice] true. /// /// The agent joins as soon as this returns. Joining the same call from this device is what /// `VoiceSession.join` is for; this only starts the agent and gives you the state. Future voice([SessionOptions options = const SessionOptions()]) => - _follow(sessions, options, voice: true); + openSession(sessions, options, voice: true); /// Follows a session this caller already has, without creating one: after a relaunch, on /// another screen, or a fork. It is also how a conversation is carried on. @@ -168,32 +170,6 @@ final class StreamAgents { _client.close(); } } - - Future _follow( - Sessions sessions, - SessionOptions options, { - required bool voice, - }) async { - final session = AgentSession( - await openSession(sessions, options, voice: voice), - backend: backend, - sessions: sessions, - tools: options.tools, - ); - try { - await session.start(); - } on AgentsException { - // The agent is in the conversation whether or not anybody here can follow it, so it is - // stopped rather than left holding a call nobody is listening to. - try { - await sessions.stop(session.id); - } on AgentsException { - // Reporting why the socket did not open matters more than this. - } - rethrow; - } - return session; - } } /// A token about to expire is as good as expired by the time it reaches the router. @@ -204,22 +180,20 @@ const _leeway = Duration(minutes: 1); /// The JavaScript SDK's `client.agent("docs")`: every session opened or listed here is this /// agent's, unless the options name another config. final class Agent { - Agent._(this._agents, this.name) - : sessions = Sessions(_agents._operations, agent: name, userId: _agents.backend.userId); - - final StreamAgents _agents; + Agent._(StreamAgents agents, this.name) + : sessions = Sessions(agents._operations, agents.backend, agent: name); /// The name its config was stored under. final String name; - /// This agent's conversations. + /// This agent's conversations. `sessions.create` opens one and follows it. final Sessions sessions; /// Holds a conversation with this agent in writing. Future chat([SessionOptions options = const SessionOptions()]) => - _agents._follow(sessions, options, voice: false); + openSession(sessions, options, voice: false); /// Puts this agent on the session's own call and follows it. Future voice([SessionOptions options = const SessionOptions()]) => - _agents._follow(sessions, options, voice: true); + openSession(sessions, options, voice: true); } diff --git a/sdks/dart/stream_agents/test/core/client_test.dart b/sdks/dart/stream_agents/test/core/client_test.dart index f894fd8c28..8c5d692f59 100644 --- a/sdks/dart/stream_agents/test/core/client_test.dart +++ b/sdks/dart/stream_agents/test/core/client_test.dart @@ -313,10 +313,87 @@ void main() { 'return_url': 'com.example.app:/connected', }); expect(session.id, 's1'); - expect(session.isText, isTrue); - expect(session.mode, SessionMode.text); - expect(session.modality, SessionModality.text); - expect(session.createdAt, DateTime.utc(2026, 9, 24, 15, 54, 56, 38, 55)); + expect(session.session.isText, isTrue); + expect(session.session.mode, SessionMode.text); + expect(session.session.modality, SessionModality.text); + expect(session.session.createdAt, DateTime.utc(2026, 9, 24, 15, 54, 56, 38, 55)); + await session.close(); + }); + + test('opens a session followed, its socket open, and answers its tools', () async { + router.answer('POST /v1/agents/sessions', Answer(201, sessionJson())); + + final session = await agents + .agent('support') + .sessions + .create( + SessionOptions( + tools: [ + AgentTool( + name: 'lookup_order', + description: 'Look up an order.', + run: (arguments) async => 'Order ${arguments['order_id']}: 2 wool throws.', + ), + ], + ), + ); + expect(session.isConnected, isTrue); + expect(router.upgrades.single.path, '/v1/agents/sessions/s1/events'); + final socket = await router.socket(); + socket.send( + '{"type":"tool_call","id":"c1","name":"lookup_order",' + r'"arguments":"{\"order_id\":\"A-1042\"}","request_id":"","turn_id":"t1"}', + ); + + expect(await socket.next(), { + 'type': 'tool_result', + 'tool_call_id': 'c1', + 'output': 'Order A-1042: 2 wool throws.', + 'error': '', + }); + expect( + ((router.last('POST /v1/agents/sessions').json as Map)['tools'] as List).single['name'], + 'lookup_order', + ); + await session.close(); + }); + + test('unwatched, opens no socket and declares no tool, and turns go over HTTP alone', () async { + router + ..answer('POST /v1/agents/sessions', Answer(201, sessionJson())) + ..answer('POST /v1/agents/sessions/s1/responses', Answer(202, responseJson('r1'))); + + final session = await agents + .agent('support') + .sessions + .create( + SessionOptions( + watch: false, + tools: [AgentTool(name: 'now', description: 'The time.', run: (_) async => '')], + ), + ); + final response = await session.responses.create('What are your hours?'); + + expect(response.id, 'r1'); + expect(router.upgrades, isEmpty); + expect(session.connection.value, Connection.notStarted); + expect(router.last('POST /v1/agents/sessions').json, {'agent': 'support'}); + final unwatched = throwsA( + isA().having( + (e) => e.message, + 'message', + allOf(contains('watch: false'), contains('responses.create()')), + ), + ); + expect(() => session.say('Hello.'), unwatched); + expect(session.interrupt, unwatched); + expect(session.events, unwatched); + expect(session.start, unwatched); + await expectLater( + session.addTools([AgentTool(name: 'now', description: 'The time.', run: (_) async => '')]), + unwatched, + ); + expect(router.upgrades, isEmpty); }); test('an empty return url sends none, so the sign-in goes back to the dashboard', () async { @@ -385,9 +462,11 @@ void main() { 'config_id': 'c1', 'start_voice': true, }); - expect(session.callId, 's1'); - expect(session.callType, 'agent'); - expect(session.isText, isFalse); + expect(session.session.callId, 's1'); + expect(session.session.callType, 'agent'); + expect(session.session.isText, isFalse); + expect(session.isConnected, isTrue); + await session.close(); }); test('starting and stopping voice moves a session onto its call and off it', () async { diff --git a/sdks/dart/stream_agents/test/core/live_test.dart b/sdks/dart/stream_agents/test/core/live_test.dart index 194747840f..1662d42178 100644 --- a/sdks/dart/stream_agents/test/core/live_test.dart +++ b/sdks/dart/stream_agents/test/core/live_test.dart @@ -212,7 +212,7 @@ void main() { returnUrl: 'io.getstream.visionagents.flutterassistant:/connected', ), ); - await sessions.delete(session.id); + await session.delete(); await expectLater( sessions.create(const SessionOptions(returnUrl: 'flutterassistant:/connected')), diff --git a/sdks/dart/stream_agents/test/core/posture_test.dart b/sdks/dart/stream_agents/test/core/posture_test.dart index 5f159c061f..40927da8b7 100644 --- a/sdks/dart/stream_agents/test/core/posture_test.dart +++ b/sdks/dart/stream_agents/test/core/posture_test.dart @@ -74,7 +74,7 @@ void main() { ..answer('GET $t/t1/runs', Answer(200, pageJson([]))); final sessions = agents.sessions; - await sessions.create(); + await (await sessions.create()).close(); await sessions.query(); await sessions.search('billing'); await sessions.get('s1'); From f75131ecc8ccfa8d9574b57725073311478c10c2 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 15:07:51 -0700 Subject: [PATCH 3/5] feat(swift): sessions.create opens a watched session Co-authored-by: Cursor --- .claude/skills/sdk-swift/SKILL.md | 4 + .../app/SwiftAssistant/AgentChat.swift | 3 +- .../app/SwiftAssistant/VoiceMode.swift | 1 - .../swift_demo/app/SwiftDemo/AgentChat.swift | 3 +- .../swift_demo/app/SwiftDemo/ChatView.swift | 4 +- .../swift_demo/app/SwiftDemo/VoiceView.swift | 1 - sdks/swift/README.md | 30 +- .../Sources/StreamAgents/AgentSession.swift | 44 ++- .../Sources/StreamAgents/SessionSocket.swift | 60 +++- .../Sources/StreamAgents/StreamAgents.swift | 49 +++- .../Sources/StreamAgents/VoiceSession.swift | 9 +- .../Tests/StreamAgentsTests/LiveTests.swift | 9 +- .../StreamAgentsTests/SessionsTests.swift | 268 ++++++++++++++++++ 13 files changed, 439 insertions(+), 46 deletions(-) diff --git a/.claude/skills/sdk-swift/SKILL.md b/.claude/skills/sdk-swift/SKILL.md index b711c9c0eb..541a69e26d 100644 --- a/.claude/skills/sdk-swift/SKILL.md +++ b/.claude/skills/sdk-swift/SKILL.md @@ -247,6 +247,10 @@ let session = try await agents.agent("myagent").sessions.create() let turn = try await session.responses.create("Where is order 1042?") ``` +`sessions.create` returns once the session's socket is open, so the tools in its options are +answered from the first turn, as in JS. `watch = false` opens it with no socket; `chat` and +`voice` open one whose socket waits for `start()`. + and the advanced form is a request value, not thirty initialiser parameters: ```swift diff --git a/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift b/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift index c478d2a20c..f33ad27e47 100644 --- a/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift +++ b/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift @@ -237,7 +237,7 @@ final class AgentChat: AIConversationTransport { if isIncognito { options.incognito = true } - session = try await Assistant.agents.chat(options) + session = try await Assistant.agents.sessions.create(options) } if !isIncognito { let channel = try await session.chat() @@ -249,7 +249,6 @@ final class AgentChat: AIConversationTransport { read(channel) } follow(session) - await session.start() self.session = session publish() return session diff --git a/examples/voice_agents/swift_assistant/app/SwiftAssistant/VoiceMode.swift b/examples/voice_agents/swift_assistant/app/SwiftAssistant/VoiceMode.swift index 1800de8cfd..4893e88b97 100644 --- a/examples/voice_agents/swift_assistant/app/SwiftAssistant/VoiceMode.swift +++ b/examples/voice_agents/swift_assistant/app/SwiftAssistant/VoiceMode.swift @@ -106,7 +106,6 @@ struct VoiceMode: View { agents: Assistant.agents, sessionID: session.id) before = Set(voice.session.turns.map(\.id)) current.session = voice - await voice.session.start() await voice.join() } catch is CancellationError { } catch { diff --git a/examples/voice_agents/swift_demo/app/SwiftDemo/AgentChat.swift b/examples/voice_agents/swift_demo/app/SwiftDemo/AgentChat.swift index dae200083d..e8efd00900 100644 --- a/examples/voice_agents/swift_demo/app/SwiftDemo/AgentChat.swift +++ b/examples/voice_agents/swift_demo/app/SwiftDemo/AgentChat.swift @@ -45,7 +45,7 @@ final class AgentChat { @ObservationIgnored private let events: EventsController @ObservationIgnored private var following: Set = [] - /// Opens the session's channel and starts following the session. + /// Opens the channel of a session `sessions.create` opened, which is watched already. static func open(_ session: AgentSession) async throws -> AgentChat { let channel = try await session.chat() let chat = AgentChat(session: session, channel: channel) @@ -54,7 +54,6 @@ final class AgentChat { if let error { done.resume(throwing: error) } else { done.resume() } } } - await session.start() chat.hasOlderMessages = !channel.hasLoadedAllPreviousMessages return chat } diff --git a/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift b/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift index a0850ac5ba..0c2427beb8 100644 --- a/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift +++ b/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift @@ -32,10 +32,10 @@ struct ChatView: View { do { // A model that streams its reasoning, which is what fills the reply's thinking // panel. The call keeps the agent's own model, which answers sooner. - var options = SessionOptions(agent: agent) + var options = SessionOptions() options.llm = "xai/grok-4.7" options.tools = Demo.tools - let session = try await Demo.agents.chat(options) + let session = try await Demo.agents.agent(agent).sessions.create(options) chat = try await AgentChat.open(session) } catch is CancellationError { return diff --git a/examples/voice_agents/swift_demo/app/SwiftDemo/VoiceView.swift b/examples/voice_agents/swift_demo/app/SwiftDemo/VoiceView.swift index 09b2f8a420..c0fdae1f6c 100644 --- a/examples/voice_agents/swift_demo/app/SwiftDemo/VoiceView.swift +++ b/examples/voice_agents/swift_demo/app/SwiftDemo/VoiceView.swift @@ -34,7 +34,6 @@ struct VoiceView: View { .padding(.horizontal) } CallControls(voice: voice) - .task { await voice.session.start() } .padding(.bottom) } else { if let failure { diff --git a/sdks/swift/README.md b/sdks/swift/README.md index 4773b7b7a3..f02a14ba88 100644 --- a/sdks/swift/README.md +++ b/sdks/swift/README.md @@ -32,15 +32,20 @@ is new after that. The examples reach the package by path, so they build against let agents = StreamAgents(apiKey: "your_api_key") agents.setUser(User(id: "jlahey")) { try await yourBackend.agentToken() } -// In writing. No call is joined, nothing is transcribed or spoken. +// In writing. No call is joined, nothing is transcribed or spoken. It opens watched: the +// socket is open when this returns, so session.turns grows as the reply streams in, the +// session's tools run here, and every event is on session.events(), an AsyncStream of its own +// for each caller. let session = try await agents.agent("myagent").sessions.create() let turn = try await session.responses.create("What are your opening hours?") // session.responses.items(responseID: turn.id) reads back what the agent did. -// Or watch it arrive: session.turns grows as the reply streams in, and every event is on -// session.events(), an AsyncStream of its own for each caller. -await session.start() -_ = try await session.responses.create("And on Sundays?") +// Or with no socket at all: ask, then read the turn back. +var unwatched = SessionOptions() +unwatched.watch = false +let quiet = try await agents.agent("myagent").sessions.create(unwatched) +let asked = try await quiet.responses.create("And on Sundays?") +// quiet.say(...) and the rest of what needs the socket throw AgentsError.configuration. // Out loud. The agent joins a call and so does this device. let voice = try await VoiceSession.start(agents: agents, agent: "myagent") @@ -103,7 +108,9 @@ reply and its id: every reply numbers its steps from `r1`. The same events contr `AITypingIndicatorView`. Ask with `session.responses.create`, from `AIComposerView`'s send; its stop button is -`session.interrupt()`. Start the session first, so its socket runs this phone's tools. +`session.interrupt()`. A session from `sessions.create` is watched already, so its socket runs +this phone's tools. `agents.chat(...)` and `agents.voice(...)` open one too, without its +socket: call `start()` on it once you are reading its `events()`. An incognito session has no channel, so `chat()` throws. Show it from its socket instead: `session.turnMessages` is the conversation in the same shape, one reply per question with its @@ -111,10 +118,9 @@ steps as attachment payloads and the whole of each round of thinking. A call wai `approvals` shows its question on its step, for the session's user: ```swift -var options = SessionOptions(agent: "juno") +var options = SessionOptions() options.incognito = true -let session = try await agents.chat(options) -await session.start() +let session = try await agents.agent("juno").sessions.create(options) for message in session.turnMessages { let parts = AIMessagePart.parts(from: message.steps.map { ($0.type, $0.payload) }) @@ -138,10 +144,10 @@ let lookup = AgentTool( await Orders.local.find(arguments["order_id"]?.stringValue ?? "") } -var options = SessionOptions(agent: "myagent") +var options = SessionOptions() options.tools = [lookup] -let session = try await agents.sessions.create(options) -await session.start() // the socket is what carries tool calls to this device +// Watched once it returns: the socket is what carries tool calls to this device. +let session = try await agents.agent("myagent").sessions.create(options) ``` `executor: .client` shows the people in a persistent conversation that a device is running it, diff --git a/sdks/swift/core/Sources/StreamAgents/AgentSession.swift b/sdks/swift/core/Sources/StreamAgents/AgentSession.swift index c0430a66fa..3fcd694250 100644 --- a/sdks/swift/core/Sources/StreamAgents/AgentSession.swift +++ b/sdks/swift/core/Sources/StreamAgents/AgentSession.swift @@ -70,9 +70,14 @@ public final class AgentSession: ObservableObject { /// answer the router's `get_video_frames`. var cameraFrames: CameraFrames? - init(backend: Backend, session: Session, tools: [AgentTool]) { + /// Whether anything here watches the conversation. One opened with `watch: false` has no + /// socket: its turns go in through `responses.create` and are read back from `responses`. + private let watched: Bool + + init(backend: Backend, session: Session, tools: [AgentTool], watched: Bool = true) { self.session = session self.backend = backend + self.watched = watched self.tools = Dictionary(tools.map { ($0.name, $0) }, uniquingKeysWith: { first, _ in first }) socket = SessionSocket( url: backend.socketURL( @@ -84,9 +89,12 @@ public final class AgentSession: ObservableObject { urlSession: backend.urlSession) } - /// Opens the socket and starts following the conversation. Doing this twice does nothing. + /// Opens the socket and starts following the conversation, returning once the router is + /// sending it here, or once it failed, which `failure` says. A session from + /// `sessions.create` has started already. Doing this twice does nothing, and so does doing + /// it on a session opened with `watch: false`. public func start() async { - guard pump == nil else { return } + guard watched, pump == nil else { return } let headers: [String: String] do { headers = try await backend.headers() @@ -113,6 +121,10 @@ public final class AgentSession: ObservableObject { self?.stopped(.transport(error)) } } + // A refused upgrade ends the stream at once, with what the router answered. + if await !socket.isOpen { + await pump?.value + } } /// Every event as it arrives, for a caller building on more than `conversation` holds: @@ -143,6 +155,7 @@ public final class AgentSession: ObservableObject { /// Speaks this without going through the model. `interrupt` stops what the agent is saying /// first, rather than waiting for it to finish. public func say(_ text: String, interrupt: Bool = false) async throws { + let socket = try held() if interrupt { try await socket.send(.interrupt()) } @@ -152,7 +165,7 @@ public final class AgentSession: ObservableObject { /// Abandons the reply in flight, or with `requestID` only the one answering that request, /// wherever it got to. An event's `request_id` names it. public func interrupt(requestID: String = "") async throws { - try await socket.send(.interrupt(requestID: requestID)) + try await held().send(.interrupt(requestID: requestID)) } /// Has the agent join the session's call, `agent:`, and carry the conversation @@ -215,6 +228,8 @@ public final class AgentSession: ObservableObject { /// the tools back; after any other failure they stay answered, since the router may have /// added them, until the next add or remove of the name settles it. public func add(tools: [AgentTool]) async throws { + // A conversation nothing is watching has nowhere to be asked for a call. + _ = try held() guard !tools.isEmpty else { return } try await oneAtATime { // A name whose adding or removal was never answered may or may not be the router's: @@ -286,6 +301,7 @@ public final class AgentSession: ObservableObject { /// call that is not waiting is left alone, so answering twice does nothing. If the answer /// cannot be sent, the call waits again. public func decide(_ toolCallID: String, allowed: Bool, summary: String = "") async throws { + let socket = try held() guard let index = approvals.firstIndex(where: { $0.id == toolCallID }) else { return } let request = approvals.remove(at: index) let call = request.call @@ -313,12 +329,16 @@ public final class AgentSession: ObservableObject { /// runs nothing and the agent is told the person did not connect. A call that is not waiting /// is left alone. public func declineLogin(_ toolCallID: String) async throws { - try await socket.send(.toolApproval(id: toolCallID, allowed: false)) + try await held().send(.toolApproval(id: toolCallID, allowed: false)) } /// Ends the session and closes the socket. What it recorded and remembered is kept. public func close() async { - try? await socket.send(.close) + if watched { + try? await socket.send(.close) + } else { + try? await StreamAgents(backend: backend).close(sessionID: session.id) + } await socket.close() pump?.cancel() pump = nil @@ -335,6 +355,18 @@ public final class AgentSession: ObservableObject { await close() } + /// The socket, or what to do instead of it. Everything sent through it is a message to a + /// conversation being watched, and one opened with `watch: false` is watched by nothing. + private func held() throws -> SessionSocket { + guard watched else { + throw AgentsError.configuration( + "this conversation is not being watched, so there is nothing to send to: open it " + + "without watch: false, or use responses.create and read the turn back through " + + "responses.items") + } + return socket + } + /// Runs one change of tools at a time, in the order they were asked for. A handler is /// put in place before the router is asked and taken back when it refuses, so two changes /// interleaved across their requests could take back what the other put in place. diff --git a/sdks/swift/core/Sources/StreamAgents/SessionSocket.swift b/sdks/swift/core/Sources/StreamAgents/SessionSocket.swift index a4132b1a3c..ab49795325 100644 --- a/sdks/swift/core/Sources/StreamAgents/SessionSocket.swift +++ b/sdks/swift/core/Sources/StreamAgents/SessionSocket.swift @@ -1,4 +1,5 @@ import Foundation +import StreamCore /// The socket carrying one conversation. /// @@ -18,6 +19,8 @@ public actor SessionSocket { private let urlSession: URLSession private var task: URLSessionWebSocketTask? private var reader: Task? + /// Whether the router took the last `open`'s upgrade, so the conversation's frames reach it. + private(set) var isOpen = false public init(url: URL, headers: [String: String], urlSession: URLSession = .shared) { self.url = url @@ -25,13 +28,14 @@ public actor SessionSocket { self.urlSession = urlSession } - /// Opens the socket and starts reading. + /// Opens the socket and starts reading, returning once the upgrade is answered, either way. /// /// The stream is returned rather than exposed as a property so that it cannot be attached /// to twice, and so there is no window between opening and attaching in which an event /// could be dropped: reading does not begin until the stream exists. `headers` are added - /// to the ones this socket was made with, for credentials only known once it opens. - public func open(headers extra: [String: String] = [:]) -> AsyncThrowingStream { + /// to the ones this socket was made with, for credentials only known once it opens. A + /// refused upgrade ends the stream with what the router answered. + public func open(headers extra: [String: String] = [:]) async -> AsyncThrowingStream { var request = URLRequest(url: url) for (name, value) in headers.merging(extra, uniquingKeysWith: { _, new in new }) { request.setValue(value, forHTTPHeaderField: name) @@ -41,7 +45,10 @@ public actor SessionSocket { // A conversation's frames are a few kilobytes at most. Without a ceiling, a socket // that has gone wrong at the other end can make this process buffer without bound. task.maximumMessageSize = maximumFrameSize + let upgrade = Upgrade() + task.delegate = upgrade self.task = task + isOpen = false task.resume() // Unbounded because dropping an event would lose a tool call or the end of a reply. @@ -54,6 +61,10 @@ public actor SessionSocket { await self?.read(task: task, into: continuation) } + let opened = await upgrade.answered() + if self.task === task { + isOpen = opened + } return stream } @@ -76,6 +87,7 @@ public actor SessionSocket { reader = nil task?.cancel(with: .normalClosure, reason: nil) task = nil + isOpen = false } private func read( @@ -128,3 +140,45 @@ public actor SessionSocket { continuation.finish() } } + +/// Tells whoever opened a socket when its upgrade was answered: taken, or the task ended first. +/// The task's own delegate, so it works whatever session the app hands over. +private final class Upgrade: NSObject, URLSessionWebSocketDelegate, Sendable { + private struct State { + var opened: Bool? + var waiting: CheckedContinuation? + } + + private let state = AllocatedUnfairLock(State()) + + /// Whether the upgrade was taken, once it was answered. + func answered() async -> Bool { + await withCheckedContinuation { continuation in + let opened = state.withLock { state -> Bool? in + if state.opened == nil { state.waiting = continuation } + return state.opened + } + if let opened { continuation.resume(returning: opened) } + } + } + + func urlSession( + _ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String? + ) { + answer(true) + } + + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: (any Error)?) { + answer(false) + } + + private func answer(_ opened: Bool) { + let waiting = state.withLock { state -> CheckedContinuation? in + guard state.opened == nil else { return nil } + state.opened = opened + defer { state.waiting = nil } + return state.waiting + } + waiting?.resume(returning: opened) + } +} diff --git a/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift b/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift index 27f8804579..29cb675371 100644 --- a/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift +++ b/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift @@ -54,6 +54,10 @@ public struct SessionOptions: Sendable { public var voice: String? /// Functions of yours the agent may call, answered on this device. public var tools: [AgentTool] = [] + /// Whether to watch the conversation over its socket, which is what answers `tools` and + /// reports what the agent is doing as it happens. False opens it with no socket and no + /// tools: ask through `responses.create` and read the turn back from `responses`. + public var watch = true /// Cost labels, carried onto every request the session makes. public var tags: [String: String] = [:] /// Where a sign-in the conversation asks for goes back to. Name your app's own scheme in @@ -200,7 +204,9 @@ public struct StreamAgents: Sendable { /// Holds a conversation in writing: no call is joined, nothing is transcribed or spoken. /// /// The replies still come through the model with the same instructions, skills and - /// knowledge a call would have had, and arrive as deltas on the session's socket. + /// knowledge a call would have had, and arrive as deltas on the session's socket. The + /// socket opens on `start()`, so whatever is read from `events()` before it is every event; + /// `agent(_:).sessions.create` opens it straight away. public func chat(agent: String? = nil, tools: [AgentTool] = []) async throws -> AgentSession { var options = SessionOptions(agent: agent) options.tools = tools @@ -211,14 +217,14 @@ public struct StreamAgents: Sendable { public func chat(_ options: SessionOptions) async throws -> AgentSession { var options = options options.startVoice = false - let session = try await createSession(options) - return await AgentSession(backend: backend, session: session, tools: options.tools) + return try await open(options, start: false) } /// Puts an agent on the session's own call, `agent:`, and follows it. /// /// The agent joins as soon as this returns. Joining the same call from this device is what - /// `VoiceSession` is for; this only starts the agent and gives you the state layer. + /// `VoiceSession` is for; this only starts the agent and gives you the state layer, whose + /// socket opens on `start()`. public func voice(agent: String? = nil, tools: [AgentTool] = []) async throws -> AgentSession { var options = SessionOptions(agent: agent) options.tools = tools @@ -229,8 +235,28 @@ public struct StreamAgents: Sendable { public func voice(_ options: SessionOptions) async throws -> AgentSession { var options = options options.startVoice = true - let session = try await createSession(options) - return await AgentSession(backend: backend, session: session, tools: options.tools) + return try await open(options, start: false) + } + + /// Opens a session, and with `start` watches it before returning. One whose socket cannot + /// open is stopped rather than left with nobody watching it, and the failure thrown. + func open(_ options: SessionOptions, start: Bool) async throws -> AgentSession { + guard options.watch else { + // Unwatched, no tool is run here, so none is declared. + var options = options + options.tools = [] + return await AgentSession( + backend: backend, session: try await createSession(options), tools: [], watched: false) + } + let session = await AgentSession( + backend: backend, session: try await createSession(options), tools: options.tools) + guard start else { return session } + await session.start() + if let failure = await session.failure { + try? await close(sessionID: session.id) + throw failure + } + return session } /// Starts a session without following it, for a caller building its own state layer. @@ -356,15 +382,16 @@ public struct Sessions: Sendable { self.agent = agent } - /// Opens a conversation held in writing: no call is joined, nothing is transcribed or - /// spoken. + /// Opens a conversation and starts watching it, returning once the router is sending it + /// here: the tools in `options` are answered on this device from the first turn. It is held + /// in writing unless `options.startVoice` puts the agent on its call. /// - /// Ask it things with `responses`. Call `start()` on it to watch the reply arrive and to - /// answer the tools in `options`, which run on this device. + /// Ask it things with `responses`. With `options.watch` false it opens with no socket and + /// no tools, and what needs the socket throws, saying so. public func create(_ options: SessionOptions = SessionOptions()) async throws -> AgentSession { var options = options if options.agent == nil { options.agent = agent } - return try await agents.chat(options) + return try await agents.open(options, start: true) } /// One page of this caller's conversations, the ones that ended included: the ones diff --git a/sdks/swift/core/Sources/StreamAgents/VoiceSession.swift b/sdks/swift/core/Sources/StreamAgents/VoiceSession.swift index 81a2c0cf41..2c7a419a58 100644 --- a/sdks/swift/core/Sources/StreamAgents/VoiceSession.swift +++ b/sdks/swift/core/Sources/StreamAgents/VoiceSession.swift @@ -56,14 +56,17 @@ public final class VoiceSession: ObservableObject { /// What the camera showed lately, while it is on, for the agent to look at. private let frames = CameraFrames() - /// Starts an agent on the new session's own call, `agent:`, and prepares to - /// join it. + /// Starts an agent on the new session's own call, `agent:`, follows it, and + /// prepares to join it. public static func start( agents: StreamAgents, agent: String? = nil, tools: [AgentTool] = [] ) async throws -> VoiceSession { - let session = try await agents.voice(agent: agent, tools: tools) + var options = SessionOptions(agent: agent) + options.tools = tools + options.startVoice = true + let session = try await agents.sessions.create(options) return VoiceSession(agents: agents, session: session, createdLocally: true) } diff --git a/sdks/swift/core/Tests/StreamAgentsTests/LiveTests.swift b/sdks/swift/core/Tests/StreamAgentsTests/LiveTests.swift index f7adde137a..483b0d210a 100644 --- a/sdks/swift/core/Tests/StreamAgentsTests/LiveTests.swift +++ b/sdks/swift/core/Tests/StreamAgentsTests/LiveTests.swift @@ -69,8 +69,9 @@ struct LiveTests { return "Order A-1042: 2 wool throws, 78.00, delivered 14 August, unopened." } - let session = try await Live.agents.chat(agent: Live.agent, tools: [tool]) - await session.start() + var options = SessionOptions() + options.tools = [tool] + let session = try await Live.agents.agent(Live.agent).sessions.create(options) defer { Task { await session.close() } } _ = try await session.responses.create("Look up order A-1042 and tell me what is in it.") @@ -124,7 +125,9 @@ struct LiveTests { /// Asking over HTTP gets an answer the router writes down, without the socket. @Test func askingThroughResponsesGetsAnAnswerWrittenDown() async throws { - let session = try await Live.agents.agent(Live.agent).sessions.create(SessionOptions()) + var options = SessionOptions() + options.watch = false + let session = try await Live.agents.agent(Live.agent).sessions.create(options) defer { Task { await session.close() } } let turn = try await session.responses.create("What are your opening hours? Answer in one sentence.") diff --git a/sdks/swift/core/Tests/StreamAgentsTests/SessionsTests.swift b/sdks/swift/core/Tests/StreamAgentsTests/SessionsTests.swift index e7e619cd9c..db25076fe5 100644 --- a/sdks/swift/core/Tests/StreamAgentsTests/SessionsTests.swift +++ b/sdks/swift/core/Tests/StreamAgentsTests/SessionsTests.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation import Network import Testing @@ -226,6 +227,92 @@ import Testing #expect(request.query.contains("response_id=r1")) #expect(request.query.contains("limit=200")) } + + private static let lookup = AgentTool(name: "local_lookup", description: "Read local state") { _ in + "local result" + } + + /// A session opened is watched by the time it returns, so the router's call of one of its + /// tools is answered with nothing more asked of the caller. + @MainActor + @Test func aSessionOpenedAnswersItsToolsWithoutBeingStarted() async throws { + let server = try RouterServer() + defer { server.listener.cancel() } + let agents = StreamAgents(url: try await server.url(), customerID: "acme") + var options = SessionOptions() + options.tools = [Self.lookup] + + let session = try await agents.agent("myagent").sessions.create(options) + defer { Task { await session.close() } } + + #expect(session.isConnected) + var requests = server.requests.makeAsyncIterator() + let created = try #require(try await requests.next()) + #expect(created.path == "/v1/agents/sessions") + #expect(created.body["agent"] == .string("myagent")) + #expect(created.body["tools"]?.arrayValue?.first?["name"] == .string("local_lookup")) + let watched = try #require(try await requests.next()) + #expect(watched.method == "GET") + #expect(watched.path == "/v1/agents/sessions/s1/events") + let answer = try JSONDecoder().decode( + [String: RawJSON].self, from: try #require(try await server.commands.first { @Sendable _ in true })) + #expect(answer["type"] == .string("tool_result")) + #expect(answer["tool_call_id"] == .string("c1")) + #expect(answer["output"] == .string("local result")) + } + + /// A session opened with `watch: false` has no socket and declares no tools, so what needs + /// the socket says how to do without it, and closing it stops it over HTTP. + @MainActor + @Test func aSessionOpenedUnwatchedHasNoSocketAndSaysWhatToDoInstead() async throws { + let server = try RouterServer() + defer { server.listener.cancel() } + let agents = StreamAgents(url: try await server.url(), customerID: "acme") + var options = SessionOptions() + options.tools = [Self.lookup] + options.watch = false + + let session = try await agents.agent("myagent").sessions.create(options) + await session.start() + + #expect(!session.isConnected) + let said = await #expect(throws: AgentsError.self) { try await session.say("One moment.") } + let interrupted = await #expect(throws: AgentsError.self) { try await session.interrupt() } + for error in [said, interrupted] { + guard case .configuration(let message) = error else { + Issue.record("\(String(describing: error)) does not say how to do without the socket") + continue + } + #expect(message.contains("without watch: false")) + #expect(message.contains("responses.create")) + } + await session.close() + var requests = server.requests.makeAsyncIterator() + let created = try #require(try await requests.next()) + #expect(created.path == "/v1/agents/sessions") + #expect(created.body["tools"]?.arrayValue?.isEmpty ?? true, "no tool is declared that nothing here runs") + let stopped = try #require(try await requests.next()) + #expect(stopped.method == "POST") + #expect(stopped.path == "/v1/agents/sessions/s1/stop") + } + + /// `chat` opens a session whose socket waits for `start`, which then answers its tools. + @MainActor + @Test func aChatAnswersItsToolsOnceStarted() async throws { + let server = try RouterServer() + defer { server.listener.cancel() } + let agents = StreamAgents(url: try await server.url(), customerID: "acme") + + let session = try await agents.chat(agent: "myagent", tools: [Self.lookup]) + defer { Task { await session.close() } } + #expect(!session.isConnected) + await session.start() + + let answer = try JSONDecoder().decode( + [String: RawJSON].self, from: try #require(try await server.commands.first { @Sendable _ in true })) + #expect(answer["tool_call_id"] == .string("c1")) + #expect(answer["output"] == .string("local result")) + } } @Suite struct ScheduledTasksTests { @@ -454,3 +541,184 @@ struct SessionServer { completion: .contentProcessed { _ in connection.cancel() }) } } + +/// Both halves of a router on one port, as a session uses them: HTTP, answered with the session +/// `s1`, and the session's socket, upgraded here by hand, which asks for one call of +/// `local_lookup` once it is open and hands over every frame sent back. +private struct RouterServer { + struct Request: Sendable { + let method: String + let path: String + let body: [String: RawJSON] + } + + let listener: NWListener + /// Every request in the order it arrived, the socket's upgrade included. + let requests: AsyncThrowingStream + let commands: AsyncThrowingStream + private let addresses: AsyncThrowingStream + + init() throws { + let queue = DispatchQueue(label: "router-server") + let parameters = NWParameters.tcp + parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: .any) + let listener = try NWListener(using: parameters) + self.listener = listener + let (addresses, address) = AsyncThrowingStream.makeStream() + let (requests, request) = AsyncThrowingStream.makeStream() + let (commands, command) = AsyncThrowingStream.makeStream() + self.addresses = addresses + self.requests = requests + self.commands = commands + listener.stateUpdateHandler = { state in + switch state { + case .ready: + if let port = listener.port { + address.yield(URL(string: "http://127.0.0.1:\(port.rawValue)")!) + address.finish() + } + case .failed(let error): + address.finish(throwing: error) + request.finish(throwing: error) + command.finish(throwing: error) + default: break + } + } + listener.newConnectionHandler = { connection in + connection.start(queue: queue) + Self.read(connection, Data(), request, command) + } + queue.asyncAfter(deadline: .now() + 30) { + let error = AgentsError.unreadable("test server timed out") + address.finish(throwing: error) + request.finish(throwing: error) + command.finish(throwing: error) + } + listener.start(queue: queue) + } + + func url() async throws -> URL { + try #require(try await addresses.first(where: { @Sendable _ in true })) + } + + private static func read( + _ connection: NWConnection, _ buffer: Data, + _ request: AsyncThrowingStream.Continuation, + _ command: AsyncThrowingStream.Continuation + ) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, _, done, error in + var buffer = buffer + if let data { buffer.append(data) } + guard let end = buffer.range(of: Data("\r\n\r\n".utf8)) else { + if error == nil, !done { read(connection, buffer, request, command) } + return + } + let head = String(decoding: buffer[..= length else { + if error == nil, !done { read(connection, buffer, request, command) } + return + } + let line = head[0].split(separator: " ").map(String.init) + let path = String(line[1].split(separator: "?", maxSplits: 1)[0]) + request.yield( + Request( + method: line[0], path: path, + body: (try? JSONDecoder().decode([String: RawJSON].self, from: body)) ?? [:])) + if let key = headers["sec-websocket-key"] { + upgrade(connection, key: key, command) + } else { + respond(connection, path: path) + } + } + } + + private static func respond(_ connection: NWConnection, path: String) { + let body = + path == "/v1/agents/sessions" + ? #"{"id":"s1","agent_id":"a1","call_id":"","call_type":"agent","created_at":"2026-10-02T17:00:00Z","modality":"text","state":"live","user_id":"jlahey"}"# + : "" + let head = + body.isEmpty + ? "HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n" + : "HTTP/1.1 201 Created\r\nContent-Type: application/json\r\nContent-Length: \(body.utf8.count)\r\nConnection: close\r\n\r\n" + connection.send( + content: Data((head + body).utf8), completion: .contentProcessed { _ in connection.cancel() }) + } + + private static func upgrade( + _ connection: NWConnection, key: String, _ command: AsyncThrowingStream.Continuation + ) { + let accept = Data(Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8))) + .base64EncodedString() + let head = + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + + "Sec-WebSocket-Accept: \(accept)\r\n\r\n" + let call = #"{"type":"tool_call","id":"c1","name":"local_lookup","arguments":"{}"}"# + connection.send(content: Data(head.utf8) + frame(0x1, Data(call.utf8)), completion: .idempotent) + listen(connection, Data(), command) + } + + /// Reads the frames the session sends, answering its pings. + private static func listen( + _ connection: NWConnection, _ buffer: Data, _ command: AsyncThrowingStream.Continuation + ) { + var buffer = buffer + while let next = nextFrame(&buffer) { + let (opcode, payload) = next + switch opcode { + case 0x1, 0x2: command.yield(payload) + case 0x9: connection.send(content: frame(0xA, payload), completion: .idempotent) + case 0x8: return connection.cancel() + default: break + } + } + connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, _, done, error in + guard let data, error == nil else { return } + listen(connection, buffer + data, command) + if done { connection.cancel() } + } + } + + /// A server's frame, which is never masked. + private static func frame(_ opcode: UInt8, _ payload: Data) -> Data { + var data = Data([0x80 | opcode]) + if payload.count < 126 { + data.append(UInt8(payload.count)) + } else { + data.append(contentsOf: [126, UInt8(payload.count >> 8), UInt8(payload.count & 0xff)]) + } + return data + payload + } + + /// Takes the first whole frame off `buffer`, unmasking it, or nil while it has none. + private static func nextFrame(_ buffer: inout Data) -> (UInt8, Data)? { + let bytes = [UInt8](buffer) + guard bytes.count >= 2 else { return nil } + var length = Int(bytes[1] & 0x7f) + var offset = 2 + if length == 126 { + guard bytes.count >= 4 else { return nil } + length = Int(bytes[2]) << 8 | Int(bytes[3]) + offset = 4 + } else if length == 127 { + guard bytes.count >= 10 else { return nil } + length = bytes[2..<10].reduce(0) { $0 << 8 | Int($1) } + offset = 10 + } + let masked = bytes[1] & 0x80 != 0 + guard bytes.count >= offset + (masked ? 4 : 0) + length else { return nil } + let mask = masked ? Array(bytes[offset.. Date: Thu, 19 Nov 2026 15:11:52 -0700 Subject: [PATCH 4/5] chore(kotlin): regenerate ErrorDetail from the spec Co-authored-by: Cursor --- .../io/getstream/agents/android/core/generated/ErrorDetail.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/generated/ErrorDetail.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/generated/ErrorDetail.kt index 89135fbbbf..5f6db752af 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/generated/ErrorDetail.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/generated/ErrorDetail.kt @@ -37,7 +37,7 @@ import kotlinx.serialization.encoding.Encoder /** * * - * @param code What went wrong, for a program to branch on. Every type has a code of its own name (invalid_request, unauthenticated, forbidden, payment_required, not_found, method_not_allowed, not_acceptable, conflict, gone, payload_too_large, unsupported_media_type, rate_limited, internal_error, unavailable, timeout) that a failure has when nothing names it better. The others are validation_failed, missing_customer, missing_organization, server_side_only, not_configured (this deployment does not offer the feature), modality_not_routed, unsynced_changes (a sync asked to check would write over somebody's edits), name_taken (a 409: another agent config, router config or voice, or another skill of the same agent config, already has the name, so another name will do), channel_connection_taken (a 409: another live agent config binds the channel connection as fixed, and one agent answers a channel connection's messages, so the message names that config), call_unanswered (a 409: the call placed for a phone session ended with nobody answering, and the message names how: busy, no_answer, failed or canceled. Opening the session is refused, or, when the status came while the agent was joining the call, the session is ended at once and kept as ended), no_oauth_client (a 400: a consent found no OAuth client for the connector, and the message names each way the connector lets the app give it one), invalid_config (a 400: an agent config a run names does not start a session), insufficient_credits (a 402: the organization's credit is spent, so work that costs money is refused until credit is added; a model that costs nothing is still served), system_limit_reached (a 402: the organization's spend hard limit for the day or the month is reached, whatever its credit, and the message names the interval, the amount and when it resets), spend_limit_reached (a 402: one of the organization's own spend limits is spent, on the organization, an app, a key, a usage category or a team, and the message says which and when it resets, so retrying before then is refused again), user_limit_reached (a 429: a spend limit on the one end user the request is for is spent, which stops that user and no one else, and Retry-After says when its window ends), spend_check_unavailable (a 503: the router could not share what the organization is spending with the other routers for a while, so work that costs money is refused until it can; a model that costs nothing is still served, and Retry-After says to try again in a few seconds), budget_moved (a 400: a policy sent a budget, which spend limits replaced), limit_above_parent (a 400: a spend limit is above a limit of its parent in the same interval, and the message names the parent's limit), limit_exists (a 409: the subject already has a limit at that level and interval, and the message names its id), no_limit (a 404: the subject has no spend limit to reset or grant past), use_team (a 400: a category starting team: was sent, where the team field names a team), invalid_expiry (a 400: a grant's expires_at is not in the future, or is more than 366 days away), too_many_grants (a 409: the organization has 20 live grants, as many as it may), tier_limit_reached (the organization's tier limits refuse the work, and limit says which: a 403 for a call or a transfer, to a country the tier does not reach, past its minutes or recipients of the day, or past its calls at once; a 400 for an agent config whose sandbox is bigger or more open than the organization's sandbox limits allow. The message names the tier and the limit), account_suspended (a 403: the organization is suspended, and every request is refused, one for a model that costs nothing included, until it is restored; the message says to contact support), concurrency_limited (a 429: the app has as many runs in flight as it may, and Retry-After says when to try again), model_failed (a 503: the model failed or gave no answer), prompt_injection (a 400: the prompt injection screen refused the input of a run or a tool result it read, which retrying an unchanged input does not fix), image_unavailable (a 400: an image a run was handed could not be fetched or is not a JPEG, PNG, WebP or GIF, which retrying the same image does not fix), document_too_large (a 413: a file sent with a response has more pages or text than one may, alone or with the files the conversation already holds, and the message names the limit), document_unreadable (a 400: a file sent with a response could not be read, such as one with a password, one that is damaged or one with no text in it, which retrying the same file does not fix), invalid_usage_categories (a 400: the usage categories the request names, in the token's usage_categories and team claims or in X-Stream-Usage-Categories, are more than 10, one is not 1 to 64 characters of A-Z, a-z, 0-9, _ . : -, more than one names a team, or the team is over 59 characters), memory_unavailable (a 503: the memory store could not delete what was asked, so nothing that waits on it was deleted either, and the same request later can succeed), billing_not_enabled (a 404: this deployment bills nobody, or the caller's app is in no organization it bills, so there is no account to read), provider_takes_no_key (a 400: a key was put for a provider whose work is never routed on a customer's own key here, and the message names the ones that are), campaign_locked (a 409: the campaign's state does not allow the change, such as a launched campaign's sequence, or resuming one that is running), campaign_not_ready (a 400: a launch found the campaign missing what it needs to ring anybody, and the message says what), consent_required (a 400: a launch did not say the people on the campaign agreed to be called), branch_limit_reached (a 409: the conversation has as many edit and regenerate points as Stream Chat keeps for it, 100, so a regenerate or an edit is refused and the conversation is continued in a new chat; asking and reading are unaffected), branch_too_long (a 409: the branch an activation would bring back is more than 100 messages to send again, so it is continued in a new chat), branch_unavailable (a 410: the copies of a hidden branch were deleted, as a memory truncate deletes them, so it can be neither read nor brought back), switch_unfinished (a 409: a regenerate, an edit or an activation of the session failed after Stream Chat may have rewound, and finishing it still fails because Stream Chat does not answer, so the session's questions, regenerates, edits and activations are refused meanwhile; try again: each one tries to finish it first, so it clears once Stream Chat answers), and _not_found for agent_config, call, campaign, channel_account, command, connection, grant, knowledge_document, knowledge_url, limit, provider_key, router_config, scheduled_task, session, simulation, simulation_run, skill and voice. A scheduled task also has invalid_schedule (a schedule that does not read: at and in both or neither, a date, time or duration that does not parse, or until without repeat), schedule_in_past, schedule_too_far (a one-shot, or an in, more than a year ahead), invalid_timezone, timezone_unknown (no zone was given and neither the user nor the config has one), invalid_repeat, interval_too_short (a run moved to less than 15 minutes from now and earlier than the run it replaces) and too_many_tasks (a 409: the user is at their active-task limit). More may be added, so a client should expect one it does not know. + * @param code What went wrong, for a program to branch on. Every type has a code of its own name (invalid_request, unauthenticated, forbidden, payment_required, not_found, method_not_allowed, not_acceptable, conflict, gone, payload_too_large, unsupported_media_type, rate_limited, internal_error, unavailable, timeout) that a failure has when nothing names it better. The others are validation_failed, missing_customer, missing_organization, server_side_only, not_configured (this deployment does not offer the feature), modality_not_routed, unsynced_changes (a sync asked to check would write over somebody's edits), name_taken (a 409: another agent config, router config or voice, or another skill of the same agent config, already has the name, so another name will do), channel_connection_taken (a 409: another live agent config binds the channel connection as fixed, and one agent answers a channel connection's messages, so the message names that config), call_unanswered (a 409: the call placed for a phone session ended with nobody answering, and the message names how: busy, no_answer, failed or canceled. Opening the session is refused, or, when the status came while the agent was joining the call, the session is ended at once and kept as ended), no_oauth_client (a 400: a consent found no OAuth client for the connector, and the message names each way the connector lets the app give it one), invalid_config (a 400: an agent config a run names does not start a session), tool_not_configured (a 400: a session under an agent config was declared or given a function that none of the config's tool groups names, and the message names it), insufficient_credits (a 402: the organization's credit is spent, so work that costs money is refused until credit is added; a model that costs nothing is still served), system_limit_reached (a 402: the organization's spend hard limit for the day or the month is reached, whatever its credit, and the message names the interval, the amount and when it resets), spend_limit_reached (a 402: one of the organization's own spend limits is spent, on the organization, an app, a key, a usage category or a team, and the message says which and when it resets, so retrying before then is refused again), user_limit_reached (a 429: a spend limit on the one end user the request is for is spent, which stops that user and no one else, and Retry-After says when its window ends), spend_check_unavailable (a 503: the router could not share what the organization is spending with the other routers for a while, so work that costs money is refused until it can; a model that costs nothing is still served, and Retry-After says to try again in a few seconds), budget_moved (a 400: a policy sent a budget, which spend limits replaced), limit_above_parent (a 400: a spend limit is above a limit of its parent in the same interval, and the message names the parent's limit), limit_exists (a 409: the subject already has a limit at that level and interval, and the message names its id), no_limit (a 404: the subject has no spend limit to reset or grant past), use_team (a 400: a category starting team: was sent, where the team field names a team), invalid_expiry (a 400: a grant's expires_at is not in the future, or is more than 366 days away), too_many_grants (a 409: the organization has 20 live grants, as many as it may), tier_limit_reached (the organization's tier limits refuse the work, and limit says which: a 403 for a call or a transfer, to a country the tier does not reach, past its minutes or recipients of the day, or past its calls at once; a 400 for an agent config whose sandbox is bigger or more open than the organization's sandbox limits allow. The message names the tier and the limit), account_suspended (a 403: the organization is suspended, and every request is refused, one for a model that costs nothing included, until it is restored; the message says to contact support), concurrency_limited (a 429: the app has as many runs in flight as it may, and Retry-After says when to try again), model_failed (a 503: the model failed or gave no answer), prompt_injection (a 400: the prompt injection screen refused the input of a run or a tool result it read, which retrying an unchanged input does not fix), image_unavailable (a 400: an image a run was handed could not be fetched or is not a JPEG, PNG, WebP or GIF, which retrying the same image does not fix), document_too_large (a 413: a file sent with a response has more pages or text than one may, alone or with the files the conversation already holds, and the message names the limit), document_unreadable (a 400: a file sent with a response could not be read, such as one with a password, one that is damaged or one with no text in it, which retrying the same file does not fix), invalid_usage_categories (a 400: the usage categories the request names, in the token's usage_categories and team claims or in X-Stream-Usage-Categories, are more than 10, one is not 1 to 64 characters of A-Z, a-z, 0-9, _ . : -, more than one names a team, or the team is over 59 characters), memory_unavailable (a 503: the memory store could not delete what was asked, so nothing that waits on it was deleted either, and the same request later can succeed), billing_not_enabled (a 404: this deployment bills nobody, or the caller's app is in no organization it bills, so there is no account to read), provider_takes_no_key (a 400: a key was put for a provider whose work is never routed on a customer's own key here, and the message names the ones that are), campaign_locked (a 409: the campaign's state does not allow the change, such as a launched campaign's sequence, or resuming one that is running), campaign_not_ready (a 400: a launch found the campaign missing what it needs to ring anybody, and the message says what), consent_required (a 400: a launch did not say the people on the campaign agreed to be called), branch_limit_reached (a 409: the conversation has as many edit and regenerate points as Stream Chat keeps for it, 100, so a regenerate or an edit is refused and the conversation is continued in a new chat; asking and reading are unaffected), branch_too_long (a 409: the branch an activation would bring back is more than 100 messages to send again, so it is continued in a new chat), branch_unavailable (a 410: the copies of a hidden branch were deleted, as a memory truncate deletes them, so it can be neither read nor brought back), switch_unfinished (a 409: a regenerate, an edit or an activation of the session failed after Stream Chat may have rewound, and finishing it still fails because Stream Chat does not answer, so the session's questions, regenerates, edits and activations are refused meanwhile; try again: each one tries to finish it first, so it clears once Stream Chat answers), and _not_found for agent_config, call, campaign, channel_account, command, connection, grant, knowledge_document, knowledge_url, limit, provider_key, router_config, scheduled_task, session, simulation, simulation_run, skill and voice. A scheduled task also has invalid_schedule (a schedule that does not read: at and in both or neither, a date, time or duration that does not parse, or until without repeat), schedule_in_past, schedule_too_far (a one-shot, or an in, more than a year ahead), invalid_timezone, timezone_unknown (no zone was given and neither the user nor the config has one), invalid_repeat, interval_too_short (a run moved to less than 15 minutes from now and earlier than the run it replaces) and too_many_tasks (a 409: the user is at their active-task limit). More may be added, so a client should expect one it does not know. * @param docUrl Where the code is explained. * @param message What went wrong, for a person to read. Its wording may change; branch on code. * @param type @@ -47,7 +47,7 @@ import kotlinx.serialization.encoding.Encoder internal data class ErrorDetail ( - /* What went wrong, for a program to branch on. Every type has a code of its own name (invalid_request, unauthenticated, forbidden, payment_required, not_found, method_not_allowed, not_acceptable, conflict, gone, payload_too_large, unsupported_media_type, rate_limited, internal_error, unavailable, timeout) that a failure has when nothing names it better. The others are validation_failed, missing_customer, missing_organization, server_side_only, not_configured (this deployment does not offer the feature), modality_not_routed, unsynced_changes (a sync asked to check would write over somebody's edits), name_taken (a 409: another agent config, router config or voice, or another skill of the same agent config, already has the name, so another name will do), channel_connection_taken (a 409: another live agent config binds the channel connection as fixed, and one agent answers a channel connection's messages, so the message names that config), call_unanswered (a 409: the call placed for a phone session ended with nobody answering, and the message names how: busy, no_answer, failed or canceled. Opening the session is refused, or, when the status came while the agent was joining the call, the session is ended at once and kept as ended), no_oauth_client (a 400: a consent found no OAuth client for the connector, and the message names each way the connector lets the app give it one), invalid_config (a 400: an agent config a run names does not start a session), insufficient_credits (a 402: the organization's credit is spent, so work that costs money is refused until credit is added; a model that costs nothing is still served), system_limit_reached (a 402: the organization's spend hard limit for the day or the month is reached, whatever its credit, and the message names the interval, the amount and when it resets), spend_limit_reached (a 402: one of the organization's own spend limits is spent, on the organization, an app, a key, a usage category or a team, and the message says which and when it resets, so retrying before then is refused again), user_limit_reached (a 429: a spend limit on the one end user the request is for is spent, which stops that user and no one else, and Retry-After says when its window ends), spend_check_unavailable (a 503: the router could not share what the organization is spending with the other routers for a while, so work that costs money is refused until it can; a model that costs nothing is still served, and Retry-After says to try again in a few seconds), budget_moved (a 400: a policy sent a budget, which spend limits replaced), limit_above_parent (a 400: a spend limit is above a limit of its parent in the same interval, and the message names the parent's limit), limit_exists (a 409: the subject already has a limit at that level and interval, and the message names its id), no_limit (a 404: the subject has no spend limit to reset or grant past), use_team (a 400: a category starting team: was sent, where the team field names a team), invalid_expiry (a 400: a grant's expires_at is not in the future, or is more than 366 days away), too_many_grants (a 409: the organization has 20 live grants, as many as it may), tier_limit_reached (the organization's tier limits refuse the work, and limit says which: a 403 for a call or a transfer, to a country the tier does not reach, past its minutes or recipients of the day, or past its calls at once; a 400 for an agent config whose sandbox is bigger or more open than the organization's sandbox limits allow. The message names the tier and the limit), account_suspended (a 403: the organization is suspended, and every request is refused, one for a model that costs nothing included, until it is restored; the message says to contact support), concurrency_limited (a 429: the app has as many runs in flight as it may, and Retry-After says when to try again), model_failed (a 503: the model failed or gave no answer), prompt_injection (a 400: the prompt injection screen refused the input of a run or a tool result it read, which retrying an unchanged input does not fix), image_unavailable (a 400: an image a run was handed could not be fetched or is not a JPEG, PNG, WebP or GIF, which retrying the same image does not fix), document_too_large (a 413: a file sent with a response has more pages or text than one may, alone or with the files the conversation already holds, and the message names the limit), document_unreadable (a 400: a file sent with a response could not be read, such as one with a password, one that is damaged or one with no text in it, which retrying the same file does not fix), invalid_usage_categories (a 400: the usage categories the request names, in the token's usage_categories and team claims or in X-Stream-Usage-Categories, are more than 10, one is not 1 to 64 characters of A-Z, a-z, 0-9, _ . : -, more than one names a team, or the team is over 59 characters), memory_unavailable (a 503: the memory store could not delete what was asked, so nothing that waits on it was deleted either, and the same request later can succeed), billing_not_enabled (a 404: this deployment bills nobody, or the caller's app is in no organization it bills, so there is no account to read), provider_takes_no_key (a 400: a key was put for a provider whose work is never routed on a customer's own key here, and the message names the ones that are), campaign_locked (a 409: the campaign's state does not allow the change, such as a launched campaign's sequence, or resuming one that is running), campaign_not_ready (a 400: a launch found the campaign missing what it needs to ring anybody, and the message says what), consent_required (a 400: a launch did not say the people on the campaign agreed to be called), branch_limit_reached (a 409: the conversation has as many edit and regenerate points as Stream Chat keeps for it, 100, so a regenerate or an edit is refused and the conversation is continued in a new chat; asking and reading are unaffected), branch_too_long (a 409: the branch an activation would bring back is more than 100 messages to send again, so it is continued in a new chat), branch_unavailable (a 410: the copies of a hidden branch were deleted, as a memory truncate deletes them, so it can be neither read nor brought back), switch_unfinished (a 409: a regenerate, an edit or an activation of the session failed after Stream Chat may have rewound, and finishing it still fails because Stream Chat does not answer, so the session's questions, regenerates, edits and activations are refused meanwhile; try again: each one tries to finish it first, so it clears once Stream Chat answers), and _not_found for agent_config, call, campaign, channel_account, command, connection, grant, knowledge_document, knowledge_url, limit, provider_key, router_config, scheduled_task, session, simulation, simulation_run, skill and voice. A scheduled task also has invalid_schedule (a schedule that does not read: at and in both or neither, a date, time or duration that does not parse, or until without repeat), schedule_in_past, schedule_too_far (a one-shot, or an in, more than a year ahead), invalid_timezone, timezone_unknown (no zone was given and neither the user nor the config has one), invalid_repeat, interval_too_short (a run moved to less than 15 minutes from now and earlier than the run it replaces) and too_many_tasks (a 409: the user is at their active-task limit). More may be added, so a client should expect one it does not know. */ + /* What went wrong, for a program to branch on. Every type has a code of its own name (invalid_request, unauthenticated, forbidden, payment_required, not_found, method_not_allowed, not_acceptable, conflict, gone, payload_too_large, unsupported_media_type, rate_limited, internal_error, unavailable, timeout) that a failure has when nothing names it better. The others are validation_failed, missing_customer, missing_organization, server_side_only, not_configured (this deployment does not offer the feature), modality_not_routed, unsynced_changes (a sync asked to check would write over somebody's edits), name_taken (a 409: another agent config, router config or voice, or another skill of the same agent config, already has the name, so another name will do), channel_connection_taken (a 409: another live agent config binds the channel connection as fixed, and one agent answers a channel connection's messages, so the message names that config), call_unanswered (a 409: the call placed for a phone session ended with nobody answering, and the message names how: busy, no_answer, failed or canceled. Opening the session is refused, or, when the status came while the agent was joining the call, the session is ended at once and kept as ended), no_oauth_client (a 400: a consent found no OAuth client for the connector, and the message names each way the connector lets the app give it one), invalid_config (a 400: an agent config a run names does not start a session), tool_not_configured (a 400: a session under an agent config was declared or given a function that none of the config's tool groups names, and the message names it), insufficient_credits (a 402: the organization's credit is spent, so work that costs money is refused until credit is added; a model that costs nothing is still served), system_limit_reached (a 402: the organization's spend hard limit for the day or the month is reached, whatever its credit, and the message names the interval, the amount and when it resets), spend_limit_reached (a 402: one of the organization's own spend limits is spent, on the organization, an app, a key, a usage category or a team, and the message says which and when it resets, so retrying before then is refused again), user_limit_reached (a 429: a spend limit on the one end user the request is for is spent, which stops that user and no one else, and Retry-After says when its window ends), spend_check_unavailable (a 503: the router could not share what the organization is spending with the other routers for a while, so work that costs money is refused until it can; a model that costs nothing is still served, and Retry-After says to try again in a few seconds), budget_moved (a 400: a policy sent a budget, which spend limits replaced), limit_above_parent (a 400: a spend limit is above a limit of its parent in the same interval, and the message names the parent's limit), limit_exists (a 409: the subject already has a limit at that level and interval, and the message names its id), no_limit (a 404: the subject has no spend limit to reset or grant past), use_team (a 400: a category starting team: was sent, where the team field names a team), invalid_expiry (a 400: a grant's expires_at is not in the future, or is more than 366 days away), too_many_grants (a 409: the organization has 20 live grants, as many as it may), tier_limit_reached (the organization's tier limits refuse the work, and limit says which: a 403 for a call or a transfer, to a country the tier does not reach, past its minutes or recipients of the day, or past its calls at once; a 400 for an agent config whose sandbox is bigger or more open than the organization's sandbox limits allow. The message names the tier and the limit), account_suspended (a 403: the organization is suspended, and every request is refused, one for a model that costs nothing included, until it is restored; the message says to contact support), concurrency_limited (a 429: the app has as many runs in flight as it may, and Retry-After says when to try again), model_failed (a 503: the model failed or gave no answer), prompt_injection (a 400: the prompt injection screen refused the input of a run or a tool result it read, which retrying an unchanged input does not fix), image_unavailable (a 400: an image a run was handed could not be fetched or is not a JPEG, PNG, WebP or GIF, which retrying the same image does not fix), document_too_large (a 413: a file sent with a response has more pages or text than one may, alone or with the files the conversation already holds, and the message names the limit), document_unreadable (a 400: a file sent with a response could not be read, such as one with a password, one that is damaged or one with no text in it, which retrying the same file does not fix), invalid_usage_categories (a 400: the usage categories the request names, in the token's usage_categories and team claims or in X-Stream-Usage-Categories, are more than 10, one is not 1 to 64 characters of A-Z, a-z, 0-9, _ . : -, more than one names a team, or the team is over 59 characters), memory_unavailable (a 503: the memory store could not delete what was asked, so nothing that waits on it was deleted either, and the same request later can succeed), billing_not_enabled (a 404: this deployment bills nobody, or the caller's app is in no organization it bills, so there is no account to read), provider_takes_no_key (a 400: a key was put for a provider whose work is never routed on a customer's own key here, and the message names the ones that are), campaign_locked (a 409: the campaign's state does not allow the change, such as a launched campaign's sequence, or resuming one that is running), campaign_not_ready (a 400: a launch found the campaign missing what it needs to ring anybody, and the message says what), consent_required (a 400: a launch did not say the people on the campaign agreed to be called), branch_limit_reached (a 409: the conversation has as many edit and regenerate points as Stream Chat keeps for it, 100, so a regenerate or an edit is refused and the conversation is continued in a new chat; asking and reading are unaffected), branch_too_long (a 409: the branch an activation would bring back is more than 100 messages to send again, so it is continued in a new chat), branch_unavailable (a 410: the copies of a hidden branch were deleted, as a memory truncate deletes them, so it can be neither read nor brought back), switch_unfinished (a 409: a regenerate, an edit or an activation of the session failed after Stream Chat may have rewound, and finishing it still fails because Stream Chat does not answer, so the session's questions, regenerates, edits and activations are refused meanwhile; try again: each one tries to finish it first, so it clears once Stream Chat answers), and _not_found for agent_config, call, campaign, channel_account, command, connection, grant, knowledge_document, knowledge_url, limit, provider_key, router_config, scheduled_task, session, simulation, simulation_run, skill and voice. A scheduled task also has invalid_schedule (a schedule that does not read: at and in both or neither, a date, time or duration that does not parse, or until without repeat), schedule_in_past, schedule_too_far (a one-shot, or an in, more than a year ahead), invalid_timezone, timezone_unknown (no zone was given and neither the user nor the config has one), invalid_repeat, interval_too_short (a run moved to less than 15 minutes from now and earlier than the run it replaces) and too_many_tasks (a 409: the user is at their active-task limit). More may be added, so a client should expect one it does not know. */ @SerialName(value = "code") val code: kotlin.String, From 57bff87c287036ec58a422fd74caa83ee484412d Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 15:44:59 -0700 Subject: [PATCH 5/5] fix(sdks): snapshot tools, share Swift startup, close on failed setup Co-authored-by: Cursor --- .../app/SwiftAssistant/AgentChat.swift | 16 +++++++++++----- .../swift_demo/app/SwiftDemo/ChatView.swift | 7 ++++++- .../stream_agents/lib/src/core/sessions.dart | 4 +++- .../getstream/agents/android/core/Sessions.kt | 3 ++- .../Sources/StreamAgents/AgentSession.swift | 17 +++++++++++++++-- .../Sources/StreamAgents/StreamAgents.swift | 4 ++++ 6 files changed, 41 insertions(+), 10 deletions(-) diff --git a/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift b/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift index f33ad27e47..da639cbf2e 100644 --- a/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift +++ b/examples/voice_agents/swift_assistant/app/SwiftAssistant/AgentChat.swift @@ -240,13 +240,19 @@ final class AgentChat: AIConversationTransport { session = try await Assistant.agents.sessions.create(options) } if !isIncognito { - let channel = try await session.chat() - try await withCheckedThrowingContinuation { (done: CheckedContinuation) in - channel.synchronize { error in - if let error { done.resume(throwing: error) } else { done.resume() } + do { + let channel = try await session.chat() + try await withCheckedThrowingContinuation { (done: CheckedContinuation) in + channel.synchronize { error in + if let error { done.resume(throwing: error) } else { done.resume() } + } } + read(channel) + } catch { + // A session opened just now has nobody else to stop it. + if sessionID == nil { await session.close() } + throw error } - read(channel) } follow(session) self.session = session diff --git a/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift b/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift index 0c2427beb8..b3435bd21d 100644 --- a/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift +++ b/examples/voice_agents/swift_demo/app/SwiftDemo/ChatView.swift @@ -36,7 +36,12 @@ struct ChatView: View { options.llm = "xai/grok-4.7" options.tools = Demo.tools let session = try await Demo.agents.agent(agent).sessions.create(options) - chat = try await AgentChat.open(session) + do { + chat = try await AgentChat.open(session) + } catch { + await session.close() + throw error + } } catch is CancellationError { return } catch { diff --git a/sdks/dart/stream_agents/lib/src/core/sessions.dart b/sdks/dart/stream_agents/lib/src/core/sessions.dart index 465f0267f0..6e0c76f1ce 100644 --- a/sdks/dart/stream_agents/lib/src/core/sessions.dart +++ b/sdks/dart/stream_agents/lib/src/core/sessions.dart @@ -162,6 +162,8 @@ final class Sessions { Responses responses(String id) => Responses._(_operations, id); Future _open(SessionOptions options, {required bool voice}) async { + // Taken before the request, so what is declared is what is answered here. + final tools = options.watch ? List.of(options.tools) : const []; final request = createRequestOf( options, startVoice: voice, @@ -172,7 +174,7 @@ final class Sessions { sessionOf(await _operations.createSession(body: request)), backend: _backend, sessions: this, - tools: options.watch ? options.tools : const [], + tools: tools, watch: options.watch, ); if (!options.watch) { diff --git a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt index 34866bf571..ca399a692c 100644 --- a/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt +++ b/sdks/kotlin/core/src/main/kotlin/io/getstream/agents/android/core/Sessions.kt @@ -35,7 +35,8 @@ public class Sessions internal constructor( * no tools, and its turns go through [AgentSession.responses]. */ public suspend fun create(options: SessionOptions = SessionOptions()): AgentSession { - val tools = if (options.watch) options.tools else emptyList() + // A copy, so what is declared is what is answered here even if the caller's list changes. + val tools = if (options.watch) options.tools.toList() else emptyList() val duplicate = tools.groupBy { it.name }.entries.firstOrNull { it.value.size > 1 } if (duplicate != null) { throw AgentsException.Configuration("${duplicate.key} is declared twice") diff --git a/sdks/swift/core/Sources/StreamAgents/AgentSession.swift b/sdks/swift/core/Sources/StreamAgents/AgentSession.swift index 3fcd694250..c73e9ce9e0 100644 --- a/sdks/swift/core/Sources/StreamAgents/AgentSession.swift +++ b/sdks/swift/core/Sources/StreamAgents/AgentSession.swift @@ -60,6 +60,8 @@ public final class AgentSession: ObservableObject { private var changingTools = false private var waitingToChangeTools: [CheckedContinuation] = [] private var pump: Task? + /// The one `start()` under way or done, which a second `start()` waits for. + private var starting: Task? /// Whoever is reading `events()`, each with a stream of their own, or nil once the session /// has stopped and there is nothing more to read. private var subscribers: [UUID: AsyncStream.Continuation]? = [:] @@ -94,16 +96,26 @@ public final class AgentSession: ObservableObject { /// `sessions.create` has started already. Doing this twice does nothing, and so does doing /// it on a session opened with `watch: false`. public func start() async { - guard watched, pump == nil else { return } + guard watched else { return } + if let starting { return await starting.value } + let starting = Task { await connect() } + self.starting = starting + await starting.value + // One that never got as far as the socket can be started again. + if pump == nil { self.starting = nil } + } + + private func connect() async { let headers: [String: String] do { headers = try await backend.headers() + } catch is CancellationError { + return } catch let error as AgentsError { return stopped(error) } catch { return stopped(.transport(error)) } - guard pump == nil else { return } let stream = await socket.open(headers: headers) isConnected = true pump = Task { [weak self] in @@ -342,6 +354,7 @@ public final class AgentSession: ObservableObject { await socket.close() pump?.cancel() pump = nil + starting = nil isConnected = false approvals = [] conversation.state = .ended diff --git a/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift b/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift index 29cb675371..7c045e63c1 100644 --- a/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift +++ b/sdks/swift/core/Sources/StreamAgents/StreamAgents.swift @@ -252,6 +252,10 @@ public struct StreamAgents: Sendable { backend: backend, session: try await createSession(options), tools: options.tools) guard start else { return session } await session.start() + if Task.isCancelled { + await session.close() + throw CancellationError() + } if let failure = await session.failure { try? await close(sessionID: session.id) throw failure