HERD-36: a message with pictures no longer stays on "Sending…"

app/src/main/kotlin/com/makefunstuff/herdr/ui/agents/ChatParts.kt

@@ -64,6 +64,7 @@ import com.makefunstuff.herdr.core.chat.ToolLine
 import com.makefunstuff.herdr.ui.code.AddedRemoved
 import androidx.compose.ui.semantics.heading
 import androidx.compose.ui.semantics.semantics
+import com.makefunstuff.herdr.core.chat.SentMatch
 import com.makefunstuff.herdr.core.host.Images
 import com.makefunstuff.herdr.ui.images.MessageImages
 import com.makefunstuff.herdr.ui.common.CodeBlock
@@ -325,7 +326,8 @@ private fun UserBubble(item: ChatItem.User, pending: Boolean) {
     var expanded by rememberSaveable(item.id) { mutableStateOf(false) }
     // The pictures it names show as pictures, over the words.
     val split = remember(item.text) { Images.split(item.text) }
-    val text = split.text
+    // Pictures the agent keeps itself are counted under the words, not named in them as Claude Code's `[Image #1]`.
+    val text = remember(item.text, item.images) { if (item.images > 0) SentMatch.withoutMarks(split.text) else split.text }
     val long = text.length > 600 || text.count { it == '\n' } > 12
     Row(Modifier.fillMaxWidth().padding(start = 48.dp), horizontalArrangement = Arrangement.End) {
         // Your turn: a neutral raised card (`.chat-turn-user`), never a wall of colour.

core/src/main/kotlin/com/makefunstuff/herdr/core/chat/AgentChat.kt

@@ -145,18 +145,8 @@ class AgentChat(private val host: HostSession, val paneId: String, private val s
     }
 
     /** Drops pending prompts once the transcript shows another user message like each. */
-    private fun unseen(pending: List<SentMatch.Pending>, items: List<ChatItem>): List<SentMatch.Pending> {
-        if (pending.isEmpty()) return pending
-        val users = items.filterIsInstance<ChatItem.User>()
-        val texts = users.map { it.text }
-        val now = System.currentTimeMillis()
-        return SentMatch.unconfirmed(pending) { p ->
-            // A matching message that wasn't in the transcript at send time is the new one, even
-            // when older copies have slid out of a tail-only read; the count covers ids that repeat.
-            val fresh = users.count { it.id !in p.known && SentMatch.same(p.text, it.text) }
-            maxOf(p.baseline + fresh, SentMatch.slid(p, SentMatch.count(p.text, texts), now))
-        }
-    }
+    private fun unseen(pending: List<SentMatch.Pending>, items: List<ChatItem>): List<SentMatch.Pending> =
+        SentMatch.unseen(pending, items.filterIsInstance<ChatItem.User>(), System.currentTimeMillis())
 
     private suspend fun watchScreen() {
         while (true) {

core/src/main/kotlin/com/makefunstuff/herdr/core/chat/SentMatch.kt

@@ -1,5 +1,7 @@
 package com.makefunstuff.herdr.core.chat
 
+import com.makefunstuff.herdr.core.host.Images
+
 /**
  * Matching a message the app sent to the copy the transcript, screen or board shows later.
  * Matching is per message: a sent message is confirmed by the n-th matching message beyond those
@@ -18,6 +20,35 @@ object SentMatch {
      * the other (the board appends lines, a channel truncates).
      */
     fun same(sent: String, seen: String): Boolean {
+        if (sameText(sent, seen)) return true
+        // A message with pictures goes as `@path` mentions in front of the words; the agent may record
+        // them as its own attachments instead (Claude Code's `[Image #1]`, Codex's and pi's image blocks
+        // next to the words), so it is the same message when the words are.
+        val pictures = Images.split(sent)
+        val seenWords = withoutPictures(seen)
+        // Both ways: words sent alone are those of a message the agent shows with pictures, as they count up together.
+        if (pictures.paths.isEmpty()) return seenWords != seen.trim() && sameText(sent, seenWords)
+        val words = withoutPictures(sent)
+        if (words.isBlank()) return seenWords.isBlank() || pictures.paths.any { it in seen }
+        return sameText(words, seenWords)
+    }
+
+    // Claude Code's `[Image #1]`; Codex's `[image 1024x768 PNG]`.
+    private val placeholder = Regex("\\[Image #\\d+]|\\[image (\\d+ )?\\d+x\\d+[^\\]\\n]*]")
+    private val spaces = Regex("[ \\t]{2,}")
+
+    /** [text] without the agent's own marks for its pictures, and nothing else changed: for showing it. */
+    fun withoutMarks(text: String): String {
+        val rest = text.replace(markWithSpace, "")
+        return if (rest == text) text else rest.trimEnd()
+    }
+    private val markWithSpace = Regex("(?:${placeholder.pattern})(?: |\\r?\\n)?")
+
+    /** [text]'s words, without the pictures it names (`@path`) or the agent's own marks for them. */
+    fun withoutPictures(text: String): String =
+        Images.split(text).text.replace(placeholder, " ").lines().joinToString("\n") { it.replace(spaces, " ").trim() }.trim()
+
+    private fun sameText(sent: String, seen: String): Boolean {
         val a = normalize(sent)
         val b = normalize(seen)
         if (a.isEmpty() || b.isEmpty()) return false
@@ -34,6 +65,17 @@ object SentMatch {
 
     /** Occurrences of the start of [sent] in a screen's text. */
     fun countOnScreen(sent: String, screen: String): Int {
+        val pictures = Images.split(sent)
+        val words = if (pictures.paths.isEmpty()) sent else withoutPictures(sent)
+        // Pictures alone: the agent may show them as its own marks (Claude Code's `[Image #1]`) rather than their paths.
+        // Every mark counts, also one still in its input box or of a message with words; the baseline takes those
+        // already there at the send. Good enough for the screen, the fallback until the transcript is there.
+        if (pictures.paths.isNotEmpty() && words.isBlank()) return occurrences(sent, screen) + occurrences("[Image #", screen)
+        // A message with pictures is found by its words: the agent may show its pictures its own way.
+        return occurrences(words, screen)
+    }
+
+    private fun occurrences(sent: String, screen: String): Int {
         // A terminal can wrap in the middle of a word: compare without any whitespace.
         val head = normalize(sent).filterNot { it.isWhitespace() }.take(40)
         if (head.isEmpty()) return 0
@@ -69,10 +111,27 @@ object SentMatch {
     fun unconfirmed(pending: List<Pending>, counted: (Pending) -> Int): List<Pending> {
         val nth = HashMap<String, Int>()
         return pending.filter { p ->
-            val key = normalize(p.text).take(HEAD)
+            // Picture messages are matched by their words, so they count up with the messages of those words
+            // (pictures alone, with each other): the n-th needs the n-th match.
+            val key = if (Images.split(p.text).paths.isEmpty()) normalize(p.text).take(HEAD)
+            else normalize(withoutPictures(p.text)).take(HEAD).ifEmpty { "\u0000pictures" }
             val k = (nth[key] ?: 0) + 1
             nth[key] = k
             counted(p) < p.baseline + k
         }
     }
+
+    /**
+     * The pending messages the transcript's [users] messages don't show yet. A matching message that
+     * wasn't there at send time ([Pending.known]) is the new one, even when older copies slid out of a
+     * tail-only read; the count covers ids that repeat.
+     */
+    fun unseen(pending: List<Pending>, users: List<ChatItem.User>, now: Long): List<Pending> {
+        if (pending.isEmpty()) return pending
+        val texts = users.map { it.text }
+        return unconfirmed(pending) { p ->
+            val fresh = users.count { it.id !in p.known && same(p.text, it.text) }
+            maxOf(p.baseline + fresh, slid(p, count(p.text, texts), now))
+        }
+    }
 }

core/src/test/kotlin/com/makefunstuff/herdr/core/SentMatchTest.kt

@@ -1,5 +1,7 @@
 package com.makefunstuff.herdr.core
 
+import com.makefunstuff.herdr.core.chat.ChatItem
+import com.makefunstuff.herdr.core.chat.ClaudeTranscript
 import com.makefunstuff.herdr.core.chat.SentMatch
 import com.makefunstuff.herdr.core.chat.SentMatch.Pending
 import kotlin.test.Test
@@ -76,4 +78,114 @@ class SentMatchTest {
         assertEquals(0, SentMatch.unconfirmed(listOf(p)) { SentMatch.slid(it, 3, 40_000) }.size)
         assertEquals(1, SentMatch.unconfirmed(listOf(p)) { SentMatch.slid(it, 3, 2_000) }.size)
     }
+
+    // Issue #100: a message with pictures stayed on "Sending…" when the agent recorded its pictures its own way.
+    private val up = "/home/demo/.cache/herdr-android/uploads/20261108-101204-a1b2"
+    private val words = "The Board card jumps when a machine reconnects. Keep it where it was?"
+
+    @Test fun pictureMessageAsTheMentionsSent() {
+        val sent = "@$up/shot.png $words"
+        assertEquals(0, open(listOf(Pending(sent, 0)), listOf("hi", sent)).size)
+    }
+
+    @Test fun pictureMessageAsClaudeCodesImagePlaceholders() {
+        val sent = "@$up/one.png @$up/two.png $words"
+        assertTrue(SentMatch.same(sent, "[Image #1] [Image #2] $words"))
+        assertTrue(SentMatch.same(sent, "$words [Image #3]"))
+        assertEquals(0, open(listOf(Pending(sent, 0)), listOf("[Image #1] [Image #2] $words")).size)
+    }
+
+    @Test fun pictureMessageAsWordsBesideImageBlocks() {
+        // Codex and pi keep the pictures as image blocks: the transcript's text is only the words.
+        val sent = "@$up/one.png $words"
+        assertTrue(SentMatch.same(sent, words))
+        assertEquals(0, open(listOf(Pending(sent, 0)), listOf(words)).size)
+        assertTrue(SentMatch.same(sent, "[image 1 1024x768 PNG] $words"))
+    }
+
+    @Test fun picturesOnly() {
+        val sent = "@$up/one.png @$up/two.png"
+        assertTrue(SentMatch.same(sent, "[Image #1] [Image #2]"))
+        assertTrue(SentMatch.same(sent, ""))
+        assertTrue(SentMatch.same(sent, sent))
+        // Said before: a picture message already there doesn't confirm a new one, a second one does.
+        assertEquals(1, open(listOf(Pending(sent, 1)), listOf("[Image #1]")).size)
+        assertEquals(0, open(listOf(Pending(sent, 1)), listOf("[Image #1]", "[Image #2]")).size)
+    }
+
+    @Test fun picturesAfterACommand() {
+        val sent = "/review @$up/one.png look at this"
+        assertTrue(SentMatch.same(sent, "/review [Image #1] look at this"))
+    }
+
+    @Test fun otherWordsWithPicturesDontMatch() {
+        val sent = "@$up/one.png $words"
+        assertFalse(SentMatch.same(sent, "[Image #1] something else entirely"))
+        assertFalse(SentMatch.same(sent, "[Image #1]"))
+        assertFalse(SentMatch.same("@$up/one.png", "a message without pictures"))
+        // A bracketed word is not a picture.
+        assertFalse(SentMatch.same("look", "[note] look"))
+    }
+
+    @Test fun pictureMessageOnScreen() {
+        val sent = "@$up/one.png $words"
+        assertEquals(1, SentMatch.countOnScreen(sent, "> [Image #1] $words\n\n● Looking."))
+        assertEquals(1, SentMatch.countOnScreen(sent, "> $sent"))
+    }
+
+    @Test fun picturesOnlyOnScreen() {
+        val sent = "@$up/one.png"
+        assertEquals(1, SentMatch.countOnScreen(sent, "> [Image #1]\n\n● Looking."))
+        assertEquals(1, SentMatch.countOnScreen(sent, "> $sent"))
+        assertEquals(0, SentMatch.unconfirmed(listOf(Pending(sent, 0))) { SentMatch.countOnScreen(it.text, "> [Image #1]") }.size)
+    }
+
+    @Test fun ownBracketedWordsWithPictures() {
+        // The person's own `[image of …]` is words, not the agent's mark for a picture.
+        val sent = "@$up/one.png see [image of the card] above"
+        assertTrue(SentMatch.same(sent, "[Image #1] see [image of the card] above"))
+        assertEquals("see [image of the card] above", SentMatch.withoutPictures("[Image #1] see [image of the card] above"))
+    }
+
+    @Test fun claudeTranscriptConfirmsPendingPictures() {
+        // What Claude Code writes when it attaches the pictures itself: its marks and the words, then image blocks.
+        val sent = "@$up/one.png @$up/two.png $words"
+        val t = ClaudeTranscript()
+        t.feed("""{"type":"user","uuid":"u1","message":{"role":"user","content":"Nice. Also show when each machine last answered."}}""")
+        val before = t.items().filterIsInstance<ChatItem.User>()
+        val p = Pending(sent, SentMatch.count(sent, before.map { it.text }), before.map { it.id }.toSet(), sentAt = 1_000)
+        assertEquals(1, SentMatch.unseen(listOf(p), before, now = 2_000).size)
+        val image = """{"type":"image","source":{"type":"base64","media_type":"image/png","data":""}}"""
+        t.feed("""{"type":"user","uuid":"u2","message":{"role":"user","content":[{"type":"text","text":"[Image #1] [Image #2] $words"},$image,$image]}}""")
+        val after = t.items().filterIsInstance<ChatItem.User>()
+        assertEquals(2, after.last().images)
+        assertEquals(0, SentMatch.unseen(listOf(p), after, now = 2_000).size)
+    }
+
+    @Test fun twoPictureMessagesNeedTwoMatches() {
+        val one = Pending("@$up/one.png", 0)
+        val two = Pending("@$up/two.png", 0)
+        assertEquals(listOf(two), open(listOf(one, two), listOf("[Image #1]")))
+        assertEquals(0, open(listOf(one, two), listOf("[Image #1]", "[Image #2]")).size)
+        val yesA = Pending("@$up/a.png yes", 0)
+        val yesB = Pending("@$up/b.png yes", 0)
+        assertEquals(listOf(yesB), open(listOf(yesA, yesB), listOf("[Image #1] yes")))
+    }
+
+    @Test fun marksLeaveTheRestAsWritten() {
+        assertEquals("look:\n    at(Foo.kt:12)\n  indented", SentMatch.withoutMarks("[Image #1] look:\n    at(Foo.kt:12)\n  indented"))
+        assertEquals("before after", SentMatch.withoutMarks("before [Image #2] after"))
+    }
+
+    @Test fun pictureAndPlainMessageOfTheSameWordsNeedTwoMatches() {
+        val pictures = Pending("@$up/a.png yes", 0)
+        val plain = Pending("yes", 0)
+        assertEquals(1, open(listOf(pictures, plain), listOf("yes")).size)
+        assertEquals(0, open(listOf(pictures, plain), listOf("[Image #1] yes", "yes")).size)
+    }
+
+    @Test fun marksKeepTheFirstLinesIndentation() {
+        assertEquals("    at(Foo.kt:12)", SentMatch.withoutMarks("[Image #1]\n    at(Foo.kt:12)"))
+        assertEquals("  indented, no mark", SentMatch.withoutMarks("  indented, no mark"))
+    }
 }

tools/ui-preview/src/main/kotlin/preview/Scenes.kt

@@ -172,6 +172,36 @@ private fun SceneData.replies() {
     composer = (composer ?: com.makefunstuff.herdr.data.ComposerStore.State()).copy(replies = com.makefunstuff.herdr.core.chat.QuickReplies(on = true))
 }
 
+/**
+ * devbox's app agent, at rest, takes what is sent to it as Claude Code does when it attaches the pictures
+ * itself: `[Image #1] [Image #2]` and the words, the pictures as image blocks (issue #100).
+ */
+private fun SceneData.takesPicturesAsClaude() {
+    appDone()
+    val machine = host(DEVBOX)
+    val took = { p: kotlinx.serialization.json.JsonObject ->
+        val text = p["text"]?.let { (it as? kotlinx.serialization.json.JsonPrimitive)?.content }.orEmpty()
+        val words = com.makefunstuff.herdr.core.host.Images.split(text).text
+        val line = obj(
+            "type" to "user", "uuid" to "00000000-0000-4000-8000-000000000050", "timestamp" to "2026-10-17T07:44:00.000Z",
+            "message" to obj(
+                "role" to "user",
+                "content" to kotlinx.serialization.json.JsonArray(
+                    listOf(
+                        obj("type" to "text", "text" to "[Image #1] [Image #2] $words"),
+                        obj("type" to "image", "source" to obj("type" to "base64", "media_type" to "image/png", "data" to "")),
+                        obj("type" to "image", "source" to obj("type" to "base64", "media_type" to "image/png", "data" to "")),
+                    ),
+                ),
+            ),
+        )
+        machine.append(Demo.APP_SESSION, line.toString())
+        obj()
+    }
+    machine.on("agent.prompt", took)
+    machine.on("pane.send_text", took)
+}
+
 /** The app agent's conversation after two screenshots were sent to it, which devbox still has. */
 private fun SceneData.sentPictures() {
     host(DEVBOX).transcripts = host(DEVBOX).transcripts + (Demo.APP_SESSION to Demo.appTranscriptWithPictures())
@@ -584,6 +614,11 @@ fun scenes(): List<Scene> = listOf(
         drag("The picture", 0.5f to 0.99f, 0.5f to 0.55f, within = ::screenshotIn)
     }),
     Scene("chat-sent-pictures", route = Routes.agent(DEVBOX, APP_AGENT), setup = { sentPictures() }),
+    // Sent, and its agent recorded the pictures its own way (Claude Code's `[Image #1]`): it shows as sent, not "Sending…".
+    Scene("chat-pictures-sent", route = Routes.agent(DEVBOX, APP_AGENT), setup = { pictures(); takesPicturesAsClaude() }, actions = {
+        tapLabel("Send")
+        settle()
+    }),
     Scene("picture-view", route = Routes.agent(DEVBOX, APP_AGENT), setup = { sentPictures() }, actions = {
         tapLabel("Picture 2 of 2, Screenshot_20261018_101230.png")
     }),