diff --git a/app/src/main/kotlin/com/makefunstuff/herdr/ui/agents/ChatParts.kt b/app/src/main/kotlin/com/makefunstuff/herdr/ui/agents/ChatParts.kt index 7bfe14d1..b03870af 100644 --- a/app/src/main/kotlin/com/makefunstuff/herdr/ui/agents/ChatParts.kt +++ b/app/src/main/kotlin/com/makefunstuff/herdr/ui/agents/ChatParts.kt @@ -64,6 +64,7 @@ import com.makefunstuff.herdr.core.chat.ToolLine import com.makefunstuff.herdr.ui.code.AddedRemoved import androidx.compose.ui.semantics.heading import androidx.compose.ui.semantics.semantics +import com.makefunstuff.herdr.core.chat.SentMatch import com.makefunstuff.herdr.core.host.Images import com.makefunstuff.herdr.ui.images.MessageImages import com.makefunstuff.herdr.ui.common.CodeBlock @@ -325,7 +326,8 @@ private fun UserBubble(item: ChatItem.User, pending: Boolean) { var expanded by rememberSaveable(item.id) { mutableStateOf(false) } // The pictures it names show as pictures, over the words. val split = remember(item.text) { Images.split(item.text) } - val text = split.text + // Pictures the agent keeps itself are counted under the words, not named in them as Claude Code's `[Image #1]`. + val text = remember(item.text, item.images) { if (item.images > 0) SentMatch.withoutMarks(split.text) else split.text } val long = text.length > 600 || text.count { it == '\n' } > 12 Row(Modifier.fillMaxWidth().padding(start = 48.dp), horizontalArrangement = Arrangement.End) { // Your turn: a neutral raised card (`.chat-turn-user`), never a wall of colour. diff --git a/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/AgentChat.kt b/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/AgentChat.kt index eb7d288d..ea9e6d9a 100644 --- a/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/AgentChat.kt +++ b/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/AgentChat.kt @@ -145,18 +145,8 @@ class AgentChat(private val host: HostSession, val paneId: String, private val s } /** Drops pending prompts once the transcript shows another user message like each. */ - private fun unseen(pending: List, items: List): List { - if (pending.isEmpty()) return pending - val users = items.filterIsInstance() - val texts = users.map { it.text } - val now = System.currentTimeMillis() - return SentMatch.unconfirmed(pending) { p -> - // A matching message that wasn't in the transcript at send time is the new one, even - // when older copies have slid out of a tail-only read; the count covers ids that repeat. - val fresh = users.count { it.id !in p.known && SentMatch.same(p.text, it.text) } - maxOf(p.baseline + fresh, SentMatch.slid(p, SentMatch.count(p.text, texts), now)) - } - } + private fun unseen(pending: List, items: List): List = + SentMatch.unseen(pending, items.filterIsInstance(), System.currentTimeMillis()) private suspend fun watchScreen() { while (true) { diff --git a/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/SentMatch.kt b/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/SentMatch.kt index 979918ed..8c669c2d 100644 --- a/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/SentMatch.kt +++ b/core/src/main/kotlin/com/makefunstuff/herdr/core/chat/SentMatch.kt @@ -1,5 +1,7 @@ package com.makefunstuff.herdr.core.chat +import com.makefunstuff.herdr.core.host.Images + /** * Matching a message the app sent to the copy the transcript, screen or board shows later. * Matching is per message: a sent message is confirmed by the n-th matching message beyond those @@ -18,6 +20,35 @@ object SentMatch { * the other (the board appends lines, a channel truncates). */ fun same(sent: String, seen: String): Boolean { + if (sameText(sent, seen)) return true + // A message with pictures goes as `@path` mentions in front of the words; the agent may record + // them as its own attachments instead (Claude Code's `[Image #1]`, Codex's and pi's image blocks + // next to the words), so it is the same message when the words are. + val pictures = Images.split(sent) + val seenWords = withoutPictures(seen) + // Both ways: words sent alone are those of a message the agent shows with pictures, as they count up together. + if (pictures.paths.isEmpty()) return seenWords != seen.trim() && sameText(sent, seenWords) + val words = withoutPictures(sent) + if (words.isBlank()) return seenWords.isBlank() || pictures.paths.any { it in seen } + return sameText(words, seenWords) + } + + // Claude Code's `[Image #1]`; Codex's `[image 1024x768 PNG]`. + private val placeholder = Regex("\\[Image #\\d+]|\\[image (\\d+ )?\\d+x\\d+[^\\]\\n]*]") + private val spaces = Regex("[ \\t]{2,}") + + /** [text] without the agent's own marks for its pictures, and nothing else changed: for showing it. */ + fun withoutMarks(text: String): String { + val rest = text.replace(markWithSpace, "") + return if (rest == text) text else rest.trimEnd() + } + private val markWithSpace = Regex("(?:${placeholder.pattern})(?: |\\r?\\n)?") + + /** [text]'s words, without the pictures it names (`@path`) or the agent's own marks for them. */ + fun withoutPictures(text: String): String = + Images.split(text).text.replace(placeholder, " ").lines().joinToString("\n") { it.replace(spaces, " ").trim() }.trim() + + private fun sameText(sent: String, seen: String): Boolean { val a = normalize(sent) val b = normalize(seen) if (a.isEmpty() || b.isEmpty()) return false @@ -34,6 +65,17 @@ object SentMatch { /** Occurrences of the start of [sent] in a screen's text. */ fun countOnScreen(sent: String, screen: String): Int { + val pictures = Images.split(sent) + val words = if (pictures.paths.isEmpty()) sent else withoutPictures(sent) + // Pictures alone: the agent may show them as its own marks (Claude Code's `[Image #1]`) rather than their paths. + // Every mark counts, also one still in its input box or of a message with words; the baseline takes those + // already there at the send. Good enough for the screen, the fallback until the transcript is there. + if (pictures.paths.isNotEmpty() && words.isBlank()) return occurrences(sent, screen) + occurrences("[Image #", screen) + // A message with pictures is found by its words: the agent may show its pictures its own way. + return occurrences(words, screen) + } + + private fun occurrences(sent: String, screen: String): Int { // A terminal can wrap in the middle of a word: compare without any whitespace. val head = normalize(sent).filterNot { it.isWhitespace() }.take(40) if (head.isEmpty()) return 0 @@ -69,10 +111,27 @@ object SentMatch { fun unconfirmed(pending: List, counted: (Pending) -> Int): List { val nth = HashMap() return pending.filter { p -> - val key = normalize(p.text).take(HEAD) + // Picture messages are matched by their words, so they count up with the messages of those words + // (pictures alone, with each other): the n-th needs the n-th match. + val key = if (Images.split(p.text).paths.isEmpty()) normalize(p.text).take(HEAD) + else normalize(withoutPictures(p.text)).take(HEAD).ifEmpty { "\u0000pictures" } val k = (nth[key] ?: 0) + 1 nth[key] = k counted(p) < p.baseline + k } } + + /** + * The pending messages the transcript's [users] messages don't show yet. A matching message that + * wasn't there at send time ([Pending.known]) is the new one, even when older copies slid out of a + * tail-only read; the count covers ids that repeat. + */ + fun unseen(pending: List, users: List, now: Long): List { + if (pending.isEmpty()) return pending + val texts = users.map { it.text } + return unconfirmed(pending) { p -> + val fresh = users.count { it.id !in p.known && same(p.text, it.text) } + maxOf(p.baseline + fresh, slid(p, count(p.text, texts), now)) + } + } } diff --git a/core/src/test/kotlin/com/makefunstuff/herdr/core/SentMatchTest.kt b/core/src/test/kotlin/com/makefunstuff/herdr/core/SentMatchTest.kt index e32c56f9..9790f8cf 100644 --- a/core/src/test/kotlin/com/makefunstuff/herdr/core/SentMatchTest.kt +++ b/core/src/test/kotlin/com/makefunstuff/herdr/core/SentMatchTest.kt @@ -1,5 +1,7 @@ package com.makefunstuff.herdr.core +import com.makefunstuff.herdr.core.chat.ChatItem +import com.makefunstuff.herdr.core.chat.ClaudeTranscript import com.makefunstuff.herdr.core.chat.SentMatch import com.makefunstuff.herdr.core.chat.SentMatch.Pending import kotlin.test.Test @@ -76,4 +78,114 @@ class SentMatchTest { assertEquals(0, SentMatch.unconfirmed(listOf(p)) { SentMatch.slid(it, 3, 40_000) }.size) assertEquals(1, SentMatch.unconfirmed(listOf(p)) { SentMatch.slid(it, 3, 2_000) }.size) } + + // Issue #100: a message with pictures stayed on "Sending…" when the agent recorded its pictures its own way. + private val up = "/home/demo/.cache/herdr-android/uploads/20261108-101204-a1b2" + private val words = "The Board card jumps when a machine reconnects. Keep it where it was?" + + @Test fun pictureMessageAsTheMentionsSent() { + val sent = "@$up/shot.png $words" + assertEquals(0, open(listOf(Pending(sent, 0)), listOf("hi", sent)).size) + } + + @Test fun pictureMessageAsClaudeCodesImagePlaceholders() { + val sent = "@$up/one.png @$up/two.png $words" + assertTrue(SentMatch.same(sent, "[Image #1] [Image #2] $words")) + assertTrue(SentMatch.same(sent, "$words [Image #3]")) + assertEquals(0, open(listOf(Pending(sent, 0)), listOf("[Image #1] [Image #2] $words")).size) + } + + @Test fun pictureMessageAsWordsBesideImageBlocks() { + // Codex and pi keep the pictures as image blocks: the transcript's text is only the words. + val sent = "@$up/one.png $words" + assertTrue(SentMatch.same(sent, words)) + assertEquals(0, open(listOf(Pending(sent, 0)), listOf(words)).size) + assertTrue(SentMatch.same(sent, "[image 1 1024x768 PNG] $words")) + } + + @Test fun picturesOnly() { + val sent = "@$up/one.png @$up/two.png" + assertTrue(SentMatch.same(sent, "[Image #1] [Image #2]")) + assertTrue(SentMatch.same(sent, "")) + assertTrue(SentMatch.same(sent, sent)) + // Said before: a picture message already there doesn't confirm a new one, a second one does. + assertEquals(1, open(listOf(Pending(sent, 1)), listOf("[Image #1]")).size) + assertEquals(0, open(listOf(Pending(sent, 1)), listOf("[Image #1]", "[Image #2]")).size) + } + + @Test fun picturesAfterACommand() { + val sent = "/review @$up/one.png look at this" + assertTrue(SentMatch.same(sent, "/review [Image #1] look at this")) + } + + @Test fun otherWordsWithPicturesDontMatch() { + val sent = "@$up/one.png $words" + assertFalse(SentMatch.same(sent, "[Image #1] something else entirely")) + assertFalse(SentMatch.same(sent, "[Image #1]")) + assertFalse(SentMatch.same("@$up/one.png", "a message without pictures")) + // A bracketed word is not a picture. + assertFalse(SentMatch.same("look", "[note] look")) + } + + @Test fun pictureMessageOnScreen() { + val sent = "@$up/one.png $words" + assertEquals(1, SentMatch.countOnScreen(sent, "> [Image #1] $words\n\n● Looking.")) + assertEquals(1, SentMatch.countOnScreen(sent, "> $sent")) + } + + @Test fun picturesOnlyOnScreen() { + val sent = "@$up/one.png" + assertEquals(1, SentMatch.countOnScreen(sent, "> [Image #1]\n\n● Looking.")) + assertEquals(1, SentMatch.countOnScreen(sent, "> $sent")) + assertEquals(0, SentMatch.unconfirmed(listOf(Pending(sent, 0))) { SentMatch.countOnScreen(it.text, "> [Image #1]") }.size) + } + + @Test fun ownBracketedWordsWithPictures() { + // The person's own `[image of …]` is words, not the agent's mark for a picture. + val sent = "@$up/one.png see [image of the card] above" + assertTrue(SentMatch.same(sent, "[Image #1] see [image of the card] above")) + assertEquals("see [image of the card] above", SentMatch.withoutPictures("[Image #1] see [image of the card] above")) + } + + @Test fun claudeTranscriptConfirmsPendingPictures() { + // What Claude Code writes when it attaches the pictures itself: its marks and the words, then image blocks. + val sent = "@$up/one.png @$up/two.png $words" + val t = ClaudeTranscript() + t.feed("""{"type":"user","uuid":"u1","message":{"role":"user","content":"Nice. Also show when each machine last answered."}}""") + val before = t.items().filterIsInstance() + val p = Pending(sent, SentMatch.count(sent, before.map { it.text }), before.map { it.id }.toSet(), sentAt = 1_000) + assertEquals(1, SentMatch.unseen(listOf(p), before, now = 2_000).size) + val image = """{"type":"image","source":{"type":"base64","media_type":"image/png","data":""}}""" + t.feed("""{"type":"user","uuid":"u2","message":{"role":"user","content":[{"type":"text","text":"[Image #1] [Image #2] $words"},$image,$image]}}""") + val after = t.items().filterIsInstance() + assertEquals(2, after.last().images) + assertEquals(0, SentMatch.unseen(listOf(p), after, now = 2_000).size) + } + + @Test fun twoPictureMessagesNeedTwoMatches() { + val one = Pending("@$up/one.png", 0) + val two = Pending("@$up/two.png", 0) + assertEquals(listOf(two), open(listOf(one, two), listOf("[Image #1]"))) + assertEquals(0, open(listOf(one, two), listOf("[Image #1]", "[Image #2]")).size) + val yesA = Pending("@$up/a.png yes", 0) + val yesB = Pending("@$up/b.png yes", 0) + assertEquals(listOf(yesB), open(listOf(yesA, yesB), listOf("[Image #1] yes"))) + } + + @Test fun marksLeaveTheRestAsWritten() { + assertEquals("look:\n at(Foo.kt:12)\n indented", SentMatch.withoutMarks("[Image #1] look:\n at(Foo.kt:12)\n indented")) + assertEquals("before after", SentMatch.withoutMarks("before [Image #2] after")) + } + + @Test fun pictureAndPlainMessageOfTheSameWordsNeedTwoMatches() { + val pictures = Pending("@$up/a.png yes", 0) + val plain = Pending("yes", 0) + assertEquals(1, open(listOf(pictures, plain), listOf("yes")).size) + assertEquals(0, open(listOf(pictures, plain), listOf("[Image #1] yes", "yes")).size) + } + + @Test fun marksKeepTheFirstLinesIndentation() { + assertEquals(" at(Foo.kt:12)", SentMatch.withoutMarks("[Image #1]\n at(Foo.kt:12)")) + assertEquals(" indented, no mark", SentMatch.withoutMarks(" indented, no mark")) + } } diff --git a/docs/screenshots/herd-36/after/chat-pictures-sent-light.png b/docs/screenshots/herd-36/after/chat-pictures-sent-light.png new file mode 100644 index 00000000..8c6e2505 Binary files /dev/null and b/docs/screenshots/herd-36/after/chat-pictures-sent-light.png differ diff --git a/docs/screenshots/herd-36/after/chat-pictures-sent.png b/docs/screenshots/herd-36/after/chat-pictures-sent.png new file mode 100644 index 00000000..181219b1 Binary files /dev/null and b/docs/screenshots/herd-36/after/chat-pictures-sent.png differ diff --git a/docs/screenshots/herd-36/before/chat-pictures-sent-light.png b/docs/screenshots/herd-36/before/chat-pictures-sent-light.png new file mode 100644 index 00000000..334468a8 Binary files /dev/null and b/docs/screenshots/herd-36/before/chat-pictures-sent-light.png differ diff --git a/docs/screenshots/herd-36/before/chat-pictures-sent.png b/docs/screenshots/herd-36/before/chat-pictures-sent.png new file mode 100644 index 00000000..1c1b8328 Binary files /dev/null and b/docs/screenshots/herd-36/before/chat-pictures-sent.png differ diff --git a/tools/ui-preview/src/main/kotlin/preview/Scenes.kt b/tools/ui-preview/src/main/kotlin/preview/Scenes.kt index a3eb7596..37762290 100644 --- a/tools/ui-preview/src/main/kotlin/preview/Scenes.kt +++ b/tools/ui-preview/src/main/kotlin/preview/Scenes.kt @@ -172,6 +172,36 @@ private fun SceneData.replies() { composer = (composer ?: com.makefunstuff.herdr.data.ComposerStore.State()).copy(replies = com.makefunstuff.herdr.core.chat.QuickReplies(on = true)) } +/** + * devbox's app agent, at rest, takes what is sent to it as Claude Code does when it attaches the pictures + * itself: `[Image #1] [Image #2]` and the words, the pictures as image blocks (issue #100). + */ +private fun SceneData.takesPicturesAsClaude() { + appDone() + val machine = host(DEVBOX) + val took = { p: kotlinx.serialization.json.JsonObject -> + val text = p["text"]?.let { (it as? kotlinx.serialization.json.JsonPrimitive)?.content }.orEmpty() + val words = com.makefunstuff.herdr.core.host.Images.split(text).text + val line = obj( + "type" to "user", "uuid" to "00000000-0000-4000-8000-000000000050", "timestamp" to "2026-10-17T07:44:00.000Z", + "message" to obj( + "role" to "user", + "content" to kotlinx.serialization.json.JsonArray( + listOf( + obj("type" to "text", "text" to "[Image #1] [Image #2] $words"), + obj("type" to "image", "source" to obj("type" to "base64", "media_type" to "image/png", "data" to "")), + obj("type" to "image", "source" to obj("type" to "base64", "media_type" to "image/png", "data" to "")), + ), + ), + ), + ) + machine.append(Demo.APP_SESSION, line.toString()) + obj() + } + machine.on("agent.prompt", took) + machine.on("pane.send_text", took) +} + /** The app agent's conversation after two screenshots were sent to it, which devbox still has. */ private fun SceneData.sentPictures() { host(DEVBOX).transcripts = host(DEVBOX).transcripts + (Demo.APP_SESSION to Demo.appTranscriptWithPictures()) @@ -584,6 +614,11 @@ fun scenes(): List = listOf( drag("The picture", 0.5f to 0.99f, 0.5f to 0.55f, within = ::screenshotIn) }), Scene("chat-sent-pictures", route = Routes.agent(DEVBOX, APP_AGENT), setup = { sentPictures() }), + // Sent, and its agent recorded the pictures its own way (Claude Code's `[Image #1]`): it shows as sent, not "Sending…". + Scene("chat-pictures-sent", route = Routes.agent(DEVBOX, APP_AGENT), setup = { pictures(); takesPicturesAsClaude() }, actions = { + tapLabel("Send") + settle() + }), Scene("picture-view", route = Routes.agent(DEVBOX, APP_AGENT), setup = { sentPictures() }, actions = { tapLabel("Picture 2 of 2, Screenshot_20261018_101230.png") }),