From c9ef82f098ba552425a39763e726187eecaf5e87 Mon Sep 17 00:00:00 2001 From: witbrock Date: Sat, 12 Sep 2026 20:03:51 -0400 Subject: [PATCH 1/3] Add mobile image picker and recoverable attachment uploads --- .../otter_archive_and_conversation_images.md | 49 +++++++++++ .../web/von_interface/static/js/chatTab.js | 8 +- .../static/js/test/conversationImages.test.js | 37 +++++++- .../static/js/test/imagePicker.test.js | 37 ++++++++ .../static/js/utils/conversationImages.js | 80 ++++++++++++++--- .../web/von_interface/static/styles.css | 15 +++- .../web/von_interface/templates/chat_tab.html | 6 ++ tests/backend/test_conversation_images.py | 31 +++++++ tests/browser/composerControls.cjs | 2 + tests/browser/imagePicker.cjs | 87 +++++++++++++++++++ .../chatAttachmentWorkflowInput.test.js | 30 ++++++- 11 files changed, 364 insertions(+), 18 deletions(-) create mode 100644 src/frontend/web/von_interface/static/js/test/imagePicker.test.js create mode 100644 tests/browser/imagePicker.cjs diff --git a/docs/engineering/otter_archive_and_conversation_images.md b/docs/engineering/otter_archive_and_conversation_images.md index dab794985..c9e95d2b5 100644 --- a/docs/engineering/otter_archive_and_conversation_images.md +++ b/docs/engineering/otter_archive_and_conversation_images.md @@ -204,3 +204,52 @@ handoff increments `event_handoff_failures`, leaves the meeting pending, and makes the run partial while retaining the successful preservation receipt. A retry reuses the event's durable instance when one already exists. A queued or reused workflow does not prove its later mail, deck or paper effects succeeded. + +## Mobile/PWA image picker (candidate evidence, 12 September 2026) + +The composer has an **Attach image** button beside the draft. Choose one or more +photos/files, wait for previews, enter an instruction, then send. **More actions → +Paste image** attempts the browser clipboard API; ordinary image paste into the +conversation and the general **Upload File** control use the same image pipeline. +Remove an image and choose another to replace it. Removal cancels an in-progress +browser upload and excludes any late result from the request; an already stored +source is not deleted. Failed uploads remain visible, block sending, and offer +retry/removal. Closing the picker leaves the existing draft intact. + +The picker advertises PNG, JPEG and WebP. HEIC, animated and other unsupported +images need conversion to a supported still format; the backend checks actual +bytes, dimensions and size. Recognised image filenames with an empty browser MIME +type still use that validator. Clipboard denial, cancellation, missing APIs and +an empty clipboard provide the picker as a fallback. No clipboard permission is +requested until **Paste image** is pressed. Native photo-library menus, clipboard +availability and installed-PWA behaviour depend on the device/browser and have +not been established by the local Chromium fixture below. No offline upload queue +is supplied: retry when connectivity returns. Unsent image selections are not +restored after a page reload. + +Candidate checks: + +- `tests/frontend/chatAttachmentWorkflowInput.test.js`, + `static/js/test/conversationImages.test.js` and `static/js/test/imagePicker.test.js`: + 24 tests pass, including the actual chat request builder receiving only the + replacement image ID, and the next text request receiving none. +- `tests/backend/test_conversation_images.py`: 15 tests pass, including the + authenticated upload route, byte validation, descriptor return, actor isolation, + original retrieval and provider-byte transport. +- `tests/browser/imagePicker.cjs`: production template/CSS and image/compact-composer + modules in a localhost fixture, with stubbed upload/original endpoints. Chromium + passed at 360, 412 and 1440 pixels: native file chooser event, decoded preview, + removal/replacement, 44-pixel picker control and no composer overflow. Evidence: + task-worktree `.run/image-picker/results.json` and `*-preview.png`. +- `tests/browser/composerControls.cjs`: all six desktop/touch/short-viewport + profiles pass with the added keyboard stop and unchanged one-row controls. + +These are bounded component/request-path checks, not authenticated PWA acceptance +or a live vision-model trial. The DGX browser README's attachment profile on 5013 +was unreachable on this run and is documented as bound to another task. Before +claiming the requested local PWA verification, the operator must prepare a separate +localhost AgentTest profile bound to this task checkout/revision, with synthetic +actors and disposable blob/data storage, browser-test login, and a working restart +bridge. Keep real profiles and public port 5000 untouched. Then exercise the actual +composer/upload/request/history path and PWA shell on that candidate. No deployment +was requested by this task. diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index fd1548e60..73fd92788 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -11,7 +11,7 @@ import { CONVERSATION_LAYOUT_KEY, CONVERSATION_LAYOUT_KEYS, CONVERSATION_NARROW_ import { createVoiceConversation } from './voiceConversation.js'; import { getClientContext } from './clientContext.js'; import { createDictationController } from './dictation.js'; -import { renderImageAttachments, renderImageComposer, uploadConversationImage, imagesBlocked, imageItems, takeImages, restoreImages, descriptorsForIds } from "./utils/conversationImages.js"; +import { initialiseImagePicker, renderImageAttachments, renderImageComposer, uploadConversationImage, imagesBlocked, imageItems, takeImages, restoreImages, descriptorsForIds } from "./utils/conversationImages.js"; // Chat Tab Module import { annotateTurn, fetchWithTimeout, getJsonDetailed, getUserContext, getWindowSessionId, postJson, WINDOW_SESSION_HEADER } from './apiService.js'; import { @@ -6662,7 +6662,7 @@ function updateExternalConversationUi() { : promptInput.dataset.nativePlaceholder ); } - [resetButton, uploadButton, dictateButton].forEach((button) => { + [resetButton, uploadButton, dictateButton, document.getElementById('attachImageButton'), document.getElementById('pasteImageButton')].forEach((button) => { if (button) { button.disabled = readOnly; } @@ -22020,7 +22020,7 @@ async function performUploadFilesToVon(list, uploadState) { let index = 0; for (const file of list) { - if (file.type?.startsWith('image/')) { + if (file.type?.startsWith('image/') || /\.(png|jpe?g|webp|heic|heif|gif|bmp|tiff?|svg)$/i.test(file.name || '')) { index += 1; const uploaded = await uploadConversationImage(file, uploadTargetSessionId, buildChatFetchHeaders(), refreshConversationImages); if (uploaded) successCount += 1; else failureCount += 1; @@ -34181,6 +34181,8 @@ export function initializeChatTab() { }); } + initialiseImagePicker(uploadFilesToVon, renderUploadStatus); + if (uploadFileButton && uploadFileInput) { uploadFileButton.addEventListener('click', () => { try { diff --git a/src/frontend/web/von_interface/static/js/test/conversationImages.test.js b/src/frontend/web/von_interface/static/js/test/conversationImages.test.js index 8ca495966..837908259 100644 --- a/src/frontend/web/von_interface/static/js/test/conversationImages.test.js +++ b/src/frontend/web/von_interface/static/js/test/conversationImages.test.js @@ -24,7 +24,7 @@ test('failed preparation remains visible and prevents text-only submission until expect(panel.querySelector('[role=alert]').textContent).toMatch(/still PNG/); expect(imagesBlocked('failed')).toBe(true); expect(() => takeImages('failed')).toThrow(/preparation failed/); - panel.querySelector('button').click(); + panel.querySelector('button[aria-label^=Remove]').click(); expect(imagesBlocked('failed')).toBe(false); }); @@ -51,3 +51,38 @@ test('fallback Markdown resolves exact archive citations without allowing arbitr expect(parent.querySelector('a').getAttribute('href')).toBe(`/von/api/otter-archive/artifacts/${artifact}`); expect(parent.querySelectorAll('a')).toHaveLength(1); }); + +test('removal cancels in-flight upload and ignores a late response', async () => { + let finish; + global.fetch = jest.fn(() => new Promise(resolve => { finish = resolve; })); + const panel = document.createElement('div'); + const render = () => renderImageComposer(panel, 'cancel', render); + const upload = uploadConversationImage(new File(['bytes'], 'cancel.png', {type:'image/png'}), 'cancel', {}, render); + panel.querySelector('button').click(); + expect(fetch.mock.calls[0][1].signal.aborted).toBe(true); + finish({ok:true, json:async () => ({image_attachment:{concept_id:'#V#late'}})}); + expect(await upload).toBe(false); + expect(takeImages('cancel')).toEqual([]); +}); + +test('network failure can retry without duplicating or crossing conversation bindings', async () => { + global.fetch = jest.fn().mockRejectedValueOnce(new Error('Network unavailable')).mockResolvedValueOnce({ok:true, json:async () => ({image_attachment:{concept_id:'#V#retried'}})}); + const panel = document.createElement('div'); + const render = () => renderImageComposer(panel, 'retry', render); + await uploadConversationImage(new File(['bytes'], 'retry.png', {type:'image/png'}), 'retry', {}, render); + expect(imagesBlocked('retry')).toBe(true); + await imageItems('retry')[0].retry(); + expect(imageItems('retry')).toHaveLength(1); + expect(takeImages('elsewhere')).toEqual([]); + expect(takeImages('retry')).toEqual(['#V#retried']); +}); + +test.each([ + ['large.png', 'image/png', 8 * 1024 * 1024 + 1, /8 MiB/], + ['phone.heic', 'image/heic', 1, /Convert HEIC/] +])('invalid selection %s reports actionable error before upload', async (name, type, size, message) => { + global.fetch = jest.fn(); + await uploadConversationImage(new File([new Uint8Array(size)], name, {type}), name, {}, () => {}); + expect(fetch).not.toHaveBeenCalled(); + expect(imageItems(name)[0].error).toMatch(message); +}); diff --git a/src/frontend/web/von_interface/static/js/test/imagePicker.test.js b/src/frontend/web/von_interface/static/js/test/imagePicker.test.js new file mode 100644 index 000000000..3413118ef --- /dev/null +++ b/src/frontend/web/von_interface/static/js/test/imagePicker.test.js @@ -0,0 +1,37 @@ +/** @jest-environment jsdom */ +import { initialiseImagePicker } from '../utils/conversationImages.js'; +let upload, status; +beforeEach(() => { + document.body.innerHTML = ''; + upload = jest.fn(); status = jest.fn(); + Object.defineProperty(navigator, 'clipboard', {configurable:true, value:undefined}); + initialiseImagePicker(upload, status); +}); +test('picker cancellation preserves draft; selection uses shared upload path', () => { + const input = document.querySelector('input'); + const click = jest.spyOn(input, 'click'); + document.querySelector('#attachImageButton').click(); + expect(click).toHaveBeenCalled(); + input.dispatchEvent(new Event('cancel')); + expect(upload).not.toHaveBeenCalled(); + expect(status).toHaveBeenCalledWith(expect.stringContaining('unchanged')); + const file = new File(['image'], 'photo.png', {type:'image/png'}); + Object.defineProperty(input, 'files', {value:[file]}); + input.dispatchEvent(new Event('change')); + expect(upload).toHaveBeenCalledWith([file]); + expect(input.value).toBe(''); +}); +test('missing clipboard API offers the picker', () => { + document.querySelector('#pasteImageButton').click(); + expect(status).toHaveBeenCalledWith(expect.stringContaining('Use Attach image'), 'error'); +}); +test.each(['denied', 'empty', 'image'])('clipboard %s', async mode => { + Object.defineProperty(navigator, 'clipboard', {value:{read:jest.fn(async () => { + if (mode === 'denied') throw new Error('NotAllowedError'); + return mode === 'empty' ? [] : [{types:['image/png'], getType:async () => new Blob(['image'], {type:'image/png'})}]; + })}}); + document.querySelector('#pasteImageButton').click(); + await new Promise(resolve => setTimeout(resolve, 0)); + if (mode === 'image') expect(upload.mock.calls[0][0][0].type).toBe('image/png'); + else expect(status).toHaveBeenCalledWith(expect.stringContaining('Attach image'), 'error'); +}); diff --git a/src/frontend/web/von_interface/static/js/utils/conversationImages.js b/src/frontend/web/von_interface/static/js/utils/conversationImages.js index 198b71123..a67c849c7 100644 --- a/src/frontend/web/von_interface/static/js/utils/conversationImages.js +++ b/src/frontend/web/von_interface/static/js/utils/conversationImages.js @@ -34,7 +34,7 @@ export function renderImageComposer(parent, sessionId, changed) { parent.replaceChildren(); for (const item of imageItems(sessionId)) { const card = document.createElement('div'); - card.style.cssText = 'display:inline-flex;flex-direction:column;gap:4px;margin:6px;max-width:180px'; + card.className = 'conversation-image-draft'; if (item.descriptor) renderImageAttachments(card, [item.descriptor]); const label = document.createElement('span'); label.textContent = item.error || (item.descriptor ? item.name : `Uploading ${item.name}…`); @@ -42,23 +42,79 @@ export function renderImageComposer(parent, sessionId, changed) { card.appendChild(label); const remove = document.createElement('button'); remove.type = 'button'; remove.textContent = 'Remove'; remove.setAttribute('aria-label', `Remove ${item.name}`); - remove.onclick = () => { pending.set(sessionId, imageItems(sessionId).filter(x => x !== item)); changed(); }; + remove.onclick = () => { item.controller?.abort(); pending.set(sessionId, imageItems(sessionId).filter(x => x !== item)); changed(); }; + if (item.error && item.retry) { + const retry = document.createElement('button'); + retry.type = 'button'; retry.textContent = 'Retry upload'; + retry.onclick = item.retry; card.appendChild(retry); + } card.appendChild(remove); parent.appendChild(card); } } export async function uploadConversationImage(file, sessionId, headers, changed) { const item = {name: file.name}; pending.set(sessionId, [...imageItems(sessionId), item]); changed(); - try { - if (imageItems(sessionId).length > 8) throw new Error('At most eight images per message. Remove an image before sending.'); - const form = new FormData(); form.append('file', file, file.name || 'image.png'); - const response = await fetch('/von/api/images/upload', {method:'POST', headers, body:form}); - const result = await response.json(); - if (!response.ok || !result.image_attachment) throw new Error(result.message || result.error || 'Image upload failed'); - item.descriptor = result.image_attachment; known.set(item.descriptor.concept_id, item); - } catch (error) { item.error = String(error.message || error); } - changed(); - return Boolean(item.descriptor); + const upload = async () => { + item.error = null; + item.retry = null; + item.controller = new AbortController(); + changed(); + try { + if (imageItems(sessionId).length > 8) throw new Error('At most eight images per message. Remove an image before sending.'); + if (!file.size || file.size > 8 * 1024 * 1024) throw new Error('Images must be nonempty and at most 8 MiB.'); + if (file.type && !['image/png', 'image/jpeg', 'image/webp'].includes(file.type)) { + throw new Error('Use a still PNG, JPEG or WebP image. Convert HEIC or other formats before attaching.'); + } + const form = new FormData(); form.append('file', file, file.name || 'image.png'); + const response = await fetch('/von/api/images/upload', {method:'POST', headers, body:form, signal:item.controller.signal}); + const result = await response.json().catch(() => ({})); + if (!response.ok || !result.image_attachment) throw new Error(result.message || result.error || `Image upload failed (${response.status}). Retry or remove the image.`); + // A removed upload must never reappear or become a request attachment. + if (!imageItems(sessionId).includes(item)) return false; + item.descriptor = result.image_attachment; known.set(item.descriptor.concept_id, item); + } catch (error) { + if (!imageItems(sessionId).includes(item)) return false; + item.error = String(error.message || error); + item.retry = upload; + } + changed(); + return Boolean(item.descriptor); + }; + return upload(); +} + +// Both controls use the existing session-bound upload path supplied by chatTab. +export function initialiseImagePicker(upload, status) { + const input = document.getElementById('attachImageInput'); + const button = document.getElementById('attachImageButton'); + const paste = document.getElementById('pasteImageButton'); + if (button && input) { + button.addEventListener('click', () => input.click()); + input.addEventListener('change', () => { + const files = Array.from(input.files || []); + input.value = ''; + if (files.length) void upload(files); + }); + input.addEventListener('cancel', () => status('Image selection cancelled. Your draft is unchanged.')); + } + paste?.addEventListener('click', async () => { + if (!navigator.clipboard?.read) { + status('Clipboard images are unavailable here. Use Attach image, or paste into the message field.', 'error'); + return; + } + try { + const items = await navigator.clipboard.read(); + const files = []; + for (const item of items) { + const type = item.types.find(value => value.startsWith('image/')); + if (type) files.push(new File([await item.getType(type)], `pasted-image-${files.length + 1}.${type.split('/')[1]}`, {type})); + } + if (files.length) await upload(files); + else status('No image found on the clipboard. Use Attach image to choose a photo.', 'error'); + } catch (_) { + status('Clipboard access was unavailable or cancelled. Use Attach image, or paste into the message field.', 'error'); + } + }); } export function takeImages(sessionId) { if (imagesBlocked(sessionId)) throw new Error('Image preparation failed or is still running. Remove or finish the upload before sending.'); diff --git a/src/frontend/web/von_interface/static/styles.css b/src/frontend/web/von_interface/static/styles.css index 622d9a6c6..016d4d1ab 100644 --- a/src/frontend/web/von_interface/static/styles.css +++ b/src/frontend/web/von_interface/static/styles.css @@ -11854,7 +11854,7 @@ body[data-active-tab="settingsTab"] .tab-content-area { } .chat-composer-primary-actions { - width: 96px; + width: 148px; } .chat-composer-primary-actions>button:not(.composer-icon-button) { @@ -18559,3 +18559,16 @@ body.settings-body { .unified-message-content .send-message-btn { padding: 0 12px; font-size: 14px; } .unified-message-content .message-reply-destination { margin: 0 0 4px; font-size: 12px; } } + +/* Image drafts remain within the compact mobile/PWA composer. */ +#conversationImageComposer { flex-basis: 100%; min-width: 0; width: 100%; } +#conversationImageComposer:empty { display: none; } +.conversation-image-draft { display: inline-flex; vertical-align: top; flex-direction: column; gap: 4px; margin: 6px; max-width: min(180px, calc(100% - 12px)); overflow-wrap: anywhere; } +.conversation-image-draft button { min-height: 44px; } +.conversation-image-draft .conversation-image-gallery img { max-height: 140px !important; } +@media (max-width: 800px), (max-width: 1024px) and (pointer: coarse) { + .chat-composer { grid-template-columns: 44px 44px minmax(0, 1fr) auto; } + .chat-composer #attachImageButton { grid-column: 2; grid-row: 1; width: 44px; min-width: 44px; } + .chat-composer > .prompt-input-wrapper, .chat-composer > #promptInput { grid-column: 3; } + .chat-composer #sendButton { grid-column: 4; } +} diff --git a/src/frontend/web/von_interface/templates/chat_tab.html b/src/frontend/web/von_interface/templates/chat_tab.html index 6af28542d..1b9b50549 100644 --- a/src/frontend/web/von_interface/templates/chat_tab.html +++ b/src/frontend/web/von_interface/templates/chat_tab.html @@ -236,6 +236,10 @@

Send Prompt + + +

Attach up to eight still PNG, JPEG or WebP images, each up to 8 MiB and 25 million pixels. Review previews before sending. Remove an image to replace it.

+ diff --git a/tests/backend/test_conversation_images.py b/tests/backend/test_conversation_images.py index 9ce1bdd68..094164fdf 100644 --- a/tests/backend/test_conversation_images.py +++ b/tests/backend/test_conversation_images.py @@ -302,3 +302,34 @@ def test_archive_resource_citations_resolve_only_to_authenticated_source_route() links = BeautifulSoup(html, "html.parser").find_all("a") assert links[0]["href"] == "/von/api/otter-archive/artifacts/" + artifact assert not links[1].has_attr("href") + + +def test_picker_upload_validates_bytes_and_returns_request_descriptor(monkeypatch): + from flask import Blueprint, Flask + from src.backend.server.routes.conversation_image_routes import register_image_routes + + stored = [] + + def store(**kwargs): + info = images.inspect_image(kwargs['data']) + stored.append(kwargs) + return {**info, 'concept_id': '#V#phone_image', 'filename': kwargs['filename']} + + monkeypatch.setattr(images, 'store_image', store) + app = Flask(__name__) + bp = Blueprint('picker_test', __name__) + register_image_routes(bp) + app.register_blueprint(bp, url_prefix='/von') + client = app.test_client() + assert client.post('/von/api/images/upload').status_code == 401 + with override_current_actor('#V#owner', None): + assert client.post('/von/api/images/upload').status_code == 400 + invalid = client.post('/von/api/images/upload', data={'file': (io.BytesIO(b'bad'), 'phone.png')}) + assert invalid.status_code == 400 + assert invalid.json['error'] == 'invalid_image' + response = client.post('/von/api/images/upload', data={'file': (io.BytesIO(png()), 'phone.png')}) + assert response.status_code == 201 + assert response.json['image_attachment']['concept_id'] == '#V#phone_image' + assert response.json['image_attachment']['content_type'] == 'image/png' + assert len(stored) == 1 + assert stored[0]['user_concept_id'] == '#V#owner' diff --git a/tests/browser/composerControls.cjs b/tests/browser/composerControls.cjs index 2698aa6f6..ce3234804 100644 --- a/tests/browser/composerControls.cjs +++ b/tests/browser/composerControls.cjs @@ -82,6 +82,8 @@ async function bounds(page, selector) { await page.keyboard.press('Tab'); await expect(send).toBeFocused(); await page.keyboard.press('Tab'); + await expect(page.getByRole('button', { name: 'Attach image', exact: true })).toBeFocused(); + await page.keyboard.press('Tab'); if (profile.desktop) { await expect(mic).toBeFocused(); await page.keyboard.press('Tab'); diff --git a/tests/browser/imagePicker.cjs b/tests/browser/imagePicker.cjs new file mode 100644 index 000000000..1d753c5f2 --- /dev/null +++ b/tests/browser/imagePicker.cjs @@ -0,0 +1,87 @@ +// Local production-template/module fixture; no authentication or model requests. +// Run with PLAYWRIGHT_BROWSERS_PATH=/tmp/von-playwright node tests/browser/imagePicker.cjs .run/image-picker +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); +const { chromium, expect } = require('@playwright/test'); + +const root = path.resolve(__dirname, '../../src/frontend/web/von_interface'); +const markup = fs.readFileSync(path.join(root, 'templates/chat_tab.html'), 'utf8') + .replace(/{%[\s\S]*?%}/g, '').replace(/{{[\s\S]*?}}/g, ''); +const evidence = process.argv[2]; +if (evidence) fs.mkdirSync(evidence, { recursive: true }); +const server = http.createServer((req, res) => { + if (req.url === '/') { + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end(` + ${markup}`); + return; + } + const file = path.resolve(root, `.${req.url}`); + if (!file.startsWith(`${root}/static/`) || !fs.existsSync(file)) { + res.writeHead(404).end(); + return; + } + res.setHeader('Content-Type', file.endsWith('.css') ? 'text/css' : 'text/javascript'); + res.end(fs.readFileSync(file)); +}); + + +(async () => { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const browser = await chromium.launch({headless:true}); + const results = []; + try { + for (const width of [360, 412, 1440]) { + const page = await browser.newPage({viewport:{width, height:900}, isMobile:width < 800, hasTouch:width < 800}); + let uploads = 0; + const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a2ioAAAAASUVORK5CYII=', 'base64'); + await page.route('**/von/api/images/upload', route => { + uploads++; + return route.fulfill({json:{image_attachment:{concept_id:`#V#fixture-${uploads}`, filename:'phone.png'}}}); + }); + await page.route('**/von/api/images/*/original', route => route.fulfill({contentType:'image/png', body:png})); + await page.goto(`http://127.0.0.1:${server.address().port}`); + await page.evaluate(async () => { + document.querySelector('#messagesContainer').style.display = 'none'; + document.querySelector('#scrollableField').textContent = 'Image picker acceptance draft'; + const images = await import('/static/js/utils/conversationImages.js'); + const { initialiseCompactChatComposer } = await import('/static/js/components/compactComposer.js'); + initialiseCompactChatComposer(document.querySelector('.chat-composer')); + const panel = document.createElement('div'); panel.id = 'conversationImageComposer'; + document.querySelector('.chat-composer').append(panel); + const render = () => images.renderImageComposer(panel, 'fixture', render); + images.initialiseImagePicker(async files => { + for (const file of files) await images.uploadConversationImage(file, 'fixture', {}, render); + }, message => { + document.querySelector('#uploadFileStatus').textContent = message; + document.querySelector('#chatAttachmentStatus').classList.remove('hidden'); + }); + window.fixtureImages = images; + }); + const button = page.getByRole('button', {name:'Attach image', exact:true}); + await expect(button).toBeVisible(); + const box = await button.boundingBox(); + assert(box.width >= 44 && box.height >= 44); + const chooser = page.waitForEvent('filechooser'); + await button.click(); + await (await chooser).setFiles({name:'phone.png', mimeType:'image/png', buffer:png}); + await expect(page.locator('#conversationImageComposer img')).toBeVisible(); + await expect(page.locator('#conversationImageComposer img')).toHaveJSProperty('naturalWidth', 1); + await page.getByRole('button', {name:'Remove phone.png', exact:true}).click(); + await expect(page.locator('#conversationImageComposer img')).toHaveCount(0); + await page.locator('#attachImageInput').setInputFiles({name:'replacement.png', mimeType:'image/png', buffer:png}); + await expect(page.locator('#conversationImageComposer img')).toBeVisible(); + const overflow = await page.locator('.chat-composer').evaluate(el => el.scrollWidth > el.clientWidth); + assert(!overflow, `composer overflow at ${width}`); + if (evidence) await page.screenshot({path:path.join(evidence, `${width}-preview.png`)}); + const ids = await page.evaluate(() => window.fixtureImages.takeImages('fixture')); + assert.deepEqual(ids, ['#V#fixture-2']); + results.push({width, uploads, ids, overflow}); + await page.close(); + } + if (evidence) fs.writeFileSync(path.join(evidence, 'results.json'), JSON.stringify(results, null, 2)); + console.log(JSON.stringify(results)); + } finally { await browser.close(); server.close(); } +})().catch(error => { console.error(error); server.close(); process.exitCode = 1; }); diff --git a/tests/frontend/chatAttachmentWorkflowInput.test.js b/tests/frontend/chatAttachmentWorkflowInput.test.js index 513dc32e2..23fe076e7 100644 --- a/tests/frontend/chatAttachmentWorkflowInput.test.js +++ b/tests/frontend/chatAttachmentWorkflowInput.test.js @@ -75,6 +75,34 @@ describe('chat attachment workflow input binding', () => { delete global.fetch; }); + test('phone image without MIME binds only to its next request, after preview and removal', async () => { + const { __testOnly_uploadFilesToVon, sendMessage } = require(chatTabModulePath); + const bodies = []; + let uploaded = 0; + global.fetch = jest.fn(async (url, options = {}) => { + if (url === '/von/api/images/upload') return {ok:true, json:async () => ({image_attachment:{concept_id:`#V#phone-${++uploaded}`, filename:'phone.png'}})}; + if (String(url).startsWith('/von/generate')) { + bodies.push(JSON.parse(options.body)); + return {ok:true, json:async () => ({response:'Image received.', llm_debug:{model:'test-model'}})}; + } + if (String(url).startsWith('/von/history/length')) return {ok:true, json:async () => ({history_length:0, authenticated:true})}; + if (String(url).startsWith('/von/api/render_markdown')) return {ok:true, json:async () => ({html:JSON.parse(options.body).text || ''})}; + return {ok:true, json:async () => ({})}; + }); + await __testOnly_uploadFilesToVon([new File(['png'], 'phone.png')]); + expect(document.querySelector('#conversationImageComposer img')).not.toBeNull(); + document.querySelector('#conversationImageComposer button[aria-label^=Remove]').click(); + await __testOnly_uploadFilesToVon([new File(['png'], 'replacement.png')]); + document.querySelector('#promptInput').value = 'Describe this screenshot'; + await sendMessage(); + expect(bodies[0].image_attachment_ids).toEqual(['#V#phone-2']); + expect(bodies[0]).not.toHaveProperty('workflow_inputs'); + document.querySelector('#promptInput').value = 'Now a text-only question'; + await sendMessage(); + expect(bodies[1]).not.toHaveProperty('image_attachment_ids'); + expect(fetch.mock.calls.some(([url]) => url === '/von/api/files/upload')).toBe(false); + }); + test('normal upload binds its trusted file-copy id to the next generate request', async () => { const { __testOnly_uploadFilesToVon, @@ -341,7 +369,7 @@ describe('chat attachment workflow input binding', () => { await Promise.resolve(); expect(sendButton.disabled).toBe(false); - expect(sendButton.hasAttribute('title')).toBe(false); + expect(sendButton.title).toBe('Send Prompt'); await sendMessage(); From 87f70a14f973ccedb1500b395ea554fb523dddb2 Mon Sep 17 00:00:00 2001 From: witbrock Date: Mon, 14 Sep 2026 11:19:18 -0400 Subject: [PATCH 2/3] Keep MIME-less phone images out of workflow file bindings --- .../web/von_interface/static/js/chatTab.js | 2 +- tests/browser/imagePickerAuthenticated.cjs | 136 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/browser/imagePickerAuthenticated.cjs diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index 7ad21b5ee..41ef50a3e 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -21573,7 +21573,7 @@ function refreshConversationImages() { function syncConversationAttachmentWorkflowBinding(sessionId) { const key = getPendingFileCopySessionKey(sessionId); - const last = imageItems(sessionId).filter(item => item.descriptor && !item.type.startsWith('image/')).at(-1); + const last = imageItems(sessionId).filter(item => item.descriptor && !isConversationImageFile(item)).at(-1); if (last) { pendingFileCopyConceptIdsBySession.set(key, last.descriptor.concept_id); pendingFileCopyDisplayNamesBySession.set(key, last.name); diff --git a/tests/browser/imagePickerAuthenticated.cjs b/tests/browser/imagePickerAuthenticated.cjs new file mode 100644 index 000000000..80ecab674 --- /dev/null +++ b/tests/browser/imagePickerAuthenticated.cjs @@ -0,0 +1,136 @@ +// Operator-prepared disposable AgentTest only. Actual uploads/originals and PWA +// shell; generation is intercepted to preserve the fixture's no-model boundary. +process.env.PLAYWRIGHT_BROWSERS_PATH ||= '/tmp/von-playwright'; +const { chromium, expect } = require('@playwright/test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const [url, revision, actor, output = '.run/image-picker-authenticated'] = process.argv.slice(2); +assert(url && revision && actor, 'Supply fixture URL, exact revision and synthetic actor'); +assert(['localhost', '127.0.0.1'].includes(new URL(url).hostname)); +const png = fs.readFileSync(path.join(__dirname, 'fixtures/attachment.png')); +fs.mkdirSync(output, { recursive: true }); +const report = { revision, actor, url, generation: 'intercepted; no model or persisted turn claim', profiles: [] }; +const save = () => fs.writeFileSync(path.join(output, 'results.json'), JSON.stringify(report, null, 2)); + +(async () => { + const browser = await chromium.launch({ headless: true }); + try { + report.browser = browser.version(); + for (const width of [360, 412, 1440]) { + const context = await browser.newContext({ viewport: { width, height: 900 }, isMobile: width < 800, hasTouch: width < 800 }); + const page = await context.newPage(); + page.setDefaultTimeout(20000); + const result = { width, requests: [], uploads: [] }; + report.profiles.push(result); + // Install before opening the page: this test must never invoke a model. + await page.route('**/von/generate*', async route => { + const body = route.request().postDataJSON(); + result.requests.push({ prompt: body.prompt, image_attachment_ids: body.image_attachment_ids, + session_id: body.session_id, window_session: !!route.request().headers()['x-von-window-session'] }); + await route.fulfill({ json: { response: 'Fixture request received.', success: true, + terminal_status: 'completed', session_id: body.session_id, llm_debug: { model: 'fixture-no-model' } } }); + }); + await page.goto(`${url}/von/`); + const health = await (await context.request.get(`${url}/health`)).json(); + assert.equal(health.agent_test_instance, true); + assert.equal(health.version_details.git_commit, revision); + result.served_revision = health.version_details.git_commit; + const login = page.waitForResponse(r => r.url().endsWith('/api/auth/browser-test-login') && r.request().method() === 'POST'); + await page.getByRole('button', { name: /Browser test login/i }).click(); + assert.equal((await login).status(), 200); + await expect.poll(() => page.evaluate(async () => { + const auth = await (await fetch('/von/api/auth/status')).json(); + return auth.authenticated && auth.user_concept_id; + }).catch(() => null), { timeout: 20000 }).toBe(actor); + const created = page.waitForResponse(r => r.url().endsWith('/api/session/create_chat_session') && r.request().method() === 'POST'); + await page.getByRole('button', { name: 'New conversation', exact: true }).click(); + assert.equal((await created).status(), 200); + const input = page.locator('#promptInput'); + await expect(input).toBeEnabled(); + await input.fill('Describe this screenshot\nKeep my caption'); + // Verify the actual served PWA shell and controlling worker. + result.pwa = await page.evaluate(async () => { + const manifest = await (await fetch(document.querySelector('link[rel=manifest]').href)).json(); + await navigator.serviceWorker.register('/von/service-worker.js', { scope: '/von/', updateViaCache: 'none' }); + const registration = await navigator.serviceWorker.ready; + return { display: manifest.display, start_url: manifest.start_url, scope: registration.scope, + script: registration.active.scriptURL, secure: isSecureContext }; + }); + await expect.poll(() => page.evaluate(() => !!navigator.serviceWorker.controller)).toBe(true); + assert.equal(result.pwa.display, 'standalone'); + assert.equal(result.pwa.secure, true); + const attach = page.getByRole('button', { name: 'Attach image', exact: true }); + await expect(attach).toBeVisible(); + const box = await attach.boundingBox(); + assert(box.width >= 44 && box.height >= 44); + async function upload(name, chooser = false) { + const response = page.waitForResponse(r => r.url().endsWith('/api/images/upload') && r.request().method() === 'POST'); + const file = { name, mimeType: 'image/png', buffer: png }; + if (chooser) { + const selected = page.waitForEvent('filechooser'); + await attach.click(); + await (await selected).setFiles(file); + } else await page.locator('#attachImageInput').setInputFiles(file); + const uploaded = await response; + assert.equal(uploaded.status(), 201); + const descriptor = (await uploaded.json()).image_attachment; + result.uploads.push(descriptor.concept_id); + assert.equal(descriptor.sha256, crypto.createHash('sha256').update(png).digest('hex')); + const original = await context.request.get(`${url}/von/api/images/${encodeURIComponent(descriptor.concept_id)}/original`); + assert.equal(original.status(), 200); + assert.deepEqual(await original.body(), png); + await expect(page.locator('#conversationImageComposer img')).toBeVisible(); + await expect.poll(() => page.locator('#conversationImageComposer img').evaluate(img => img.naturalWidth)).toBeGreaterThan(0); + return descriptor.concept_id; + } + const removed = await upload('phone.png', true); + await page.getByRole('button', { name: 'Remove phone.png', exact: true }).click(); + await expect(page.locator('#conversationImageComposer img')).toHaveCount(0); + const replacement = await upload('replacement.png'); + await page.locator('#attachImageInput').dispatchEvent('cancel'); + await expect(input).toHaveValue('Describe this screenshot\nKeep my caption'); + result.overflow = await page.locator('.chat-composer').evaluate(el => el.scrollWidth > el.clientWidth); + assert.equal(result.overflow, false); + await page.screenshot({ path: path.join(output, `${width}-preview.png`) }); + await page.locator('#sendButton').click(); + await expect.poll(() => result.requests.length).toBe(1); + assert.deepEqual(result.requests[0].image_attachment_ids, [replacement]); + assert(!result.requests[0].image_attachment_ids.includes(removed)); + assert.equal(result.requests[0].prompt, 'Describe this screenshot\nKeep my caption'); + assert(result.requests[0].window_session); + await expect(page.locator('#conversationImageComposer')).toBeEmpty(); + await input.fill('Next text-only request'); + await page.locator('#sendButton').click(); + await expect.poll(() => result.requests.length).toBe(2); + assert.equal(result.requests[1].image_attachment_ids, undefined); + result.original_bytes_verified = true; + result.request_binding_verified = true; + // One mobile profile exercises recovery against actual retry upload. + if (width === 360) { + await input.fill('Preserve this failed-upload draft'); + await page.route('**/von/api/images/upload', r => r.fulfill({ status: 503, json: { error: 'Fixture upload failure' } })); + await page.locator('#attachImageInput').setInputFiles({ name: 'retry.png', mimeType: 'image/png', buffer: png }); + const retry = page.getByRole('button', { name: 'Retry upload of retry.png', exact: true }); + await expect(retry).toBeVisible(); + await expect(input).toHaveValue('Preserve this failed-upload draft'); + await page.locator('#sendButton').click(); + assert.equal(result.requests.length, 2); + await page.unroute('**/von/api/images/upload'); + const retried = page.waitForResponse(r => r.url().endsWith('/api/images/upload')); + await retry.click(); + assert.equal((await retried).status(), 201); + await expect(retry).toHaveCount(0); + await page.getByRole('button', { name: 'Remove retry.png', exact: true }).click(); + result.upload_failure_recovery = true; + } + save(); + await context.close(); + } + report.passed = true; + save(); + console.log(JSON.stringify({ passed: true, revision, widths: report.profiles.map(p => p.width) })); + } catch (error) { report.error = String(error); save(); throw error; } + finally { await browser.close(); } +})().catch(error => { console.error(error); process.exitCode = 1; }); From 37bdc378703d0fc0bb312d155c6d8f5708a347b8 Mon Sep 17 00:00:00 2001 From: witbrock Date: Mon, 14 Sep 2026 11:21:53 -0400 Subject: [PATCH 3/3] Record authenticated mobile image and local PWA acceptance --- .../otter_archive_and_conversation_images.md | 89 +++++++++++-------- tests/browser/imagePickerAuthenticated.cjs | 12 ++- 2 files changed, 62 insertions(+), 39 deletions(-) diff --git a/docs/engineering/otter_archive_and_conversation_images.md b/docs/engineering/otter_archive_and_conversation_images.md index c61556bcd..75fc7cd31 100644 --- a/docs/engineering/otter_archive_and_conversation_images.md +++ b/docs/engineering/otter_archive_and_conversation_images.md @@ -217,7 +217,7 @@ makes the run partial while retaining the successful preservation receipt. A retry reuses the event's durable instance when one already exists. A queued or reused workflow does not prove its later mail, deck or paper effects succeeded. -## Mobile/PWA image picker (candidate evidence, 12 September 2026) +## Mobile/PWA image picker (local acceptance, 14 September 2026) The composer has an **Attach image** button beside the draft. Choose one or more photos/files, wait for previews, enter an instruction, then send. **More actions → @@ -231,37 +231,56 @@ retry/removal. Closing the picker leaves the existing draft intact. The picker advertises PNG, JPEG and WebP. HEIC, animated and other unsupported images need conversion to a supported still format; the backend checks actual bytes, dimensions and size. Recognised image filenames with an empty browser MIME -type still use that validator. Clipboard denial, cancellation, missing APIs and -an empty clipboard provide the picker as a fallback. No clipboard permission is -requested until **Paste image** is pressed. Native photo-library menus, clipboard -availability and installed-PWA behaviour depend on the device/browser and have -not been established by the local Chromium fixture below. No offline upload queue -is supplied: retry when connectivity returns. Unsent image selections are not -restored after a page reload. - -Candidate checks: - -- `tests/frontend/chatAttachmentWorkflowInput.test.js`, - `static/js/test/conversationImages.test.js` and `static/js/test/imagePicker.test.js`: - 24 tests pass, including the actual chat request builder receiving only the - replacement image ID, and the next text request receiving none. -- `tests/backend/test_conversation_images.py`: 15 tests pass, including the - authenticated upload route, byte validation, descriptor return, actor isolation, - original retrieval and provider-byte transport. -- `tests/browser/imagePicker.cjs`: production template/CSS and image/compact-composer - modules in a localhost fixture, with stubbed upload/original endpoints. Chromium - passed at 360, 412 and 1440 pixels: native file chooser event, decoded preview, - removal/replacement, 44-pixel picker control and no composer overflow. Evidence: - task-worktree `.run/image-picker/results.json` and `*-preview.png`. -- `tests/browser/composerControls.cjs`: all six desktop/touch/short-viewport - profiles pass with the added keyboard stop and unchanged one-row controls. - -These are bounded component/request-path checks, not authenticated PWA acceptance -or a live vision-model trial. The DGX browser README's attachment profile on 5013 -was unreachable on this run and is documented as bound to another task. Before -claiming the requested local PWA verification, the operator must prepare a separate -localhost AgentTest profile bound to this task checkout/revision, with synthetic -actors and disposable blob/data storage, browser-test login, and a working restart -bridge. Keep real profiles and public port 5000 untouched. Then exercise the actual -composer/upload/request/history path and PWA shell on that candidate. No deployment -was requested by this task. +type still use that validator and remain image attachments rather than generic +workflow file inputs. Clipboard denial, cancellation, missing APIs and an empty +clipboard provide the picker as a fallback. No clipboard permission is requested +until **Paste image** is pressed. There is no offline upload queue: retry when +connectivity returns. Unsent image selections are not restored after a page reload. + +Acceptance for PR #658, product revision +`87f70a14f973ccedb1500b395ea554fb523dddb2`: + +- 34 targeted frontend tests passed across attachment request binding, image + validation/cancellation, picker/clipboard controls, upload recovery and compact + composer behaviour. All 16 `tests/backend/test_conversation_images.py` tests + passed, covering authenticated upload/original routes, invalid bytes, actor + isolation and provider-byte transport without issuing model requests. +- `tests/browser/imagePickerAuthenticated.cjs` passed on the operator-prepared + disposable fixture at `http://127.0.0.1:5085`, using visible Browser test login + as synthetic `#V#blocked_picker658_alice` and a fresh conversation per viewport. + Chromium 153.0.8010.12 passed at 360, 412 and 1440 pixels: native chooser event, + actual uploads, SHA-256 and original-byte read-back, decoded previews, + removal/replacement, preserved caption, cancellation, 44-pixel attachment + control and no composer overflow. A failed upload disabled Send and a real + retry succeeded. The actual request builder submitted only the replacement ID + to the newly created conversation, and no image IDs on the next text request. +- The same browser loaded the served standalone PWA manifest and exercised the + composer with the actual `/von/service-worker.js` controlling `/von/`. This is + local PWA-shell acceptance, not physical-device installation or offline upload. +- `tests/browser/imagePicker.cjs` passed at three widths; + `tests/browser/composerControls.cjs` passed all six desktop/touch/short profiles. + `tests/browser/mobileScreenshotPaste.cjs` passed at three widths with real + desktop clipboard grants/denial, preserved captions, ordinary multiline text + paste and stubbed upload endpoints. Authenticated and component evidence is + retained under the task worktree's `.run/image-picker-authenticated/` and + `.run/image-picker-current/` respectively. + +The authenticated fixture deliberately disables generation. Browser submissions +were intercepted at `/von/generate`; this does not establish a model answer or +persisted conversation history. It verifies actual upload/storage/read-back and +frontend request association, complemented by the targeted backend tests. + +The operator's Android 17 emulator evidence used Chrome 145 and Gboard 17.2, +with native Copy/Paste image taps, a real clipboard permission prompt and native +Allow. The unchanged PR658 helper returned a PNG that decoded to 24×16 and +preserved captions. Source and clipboard PNG hashes differ because Chromium +re-encodes clipboard images. The evidence archive is file copy +`#V#computer_file_copy_7c09b5898bec45148912a9c8bb53a996`, SHA-256 +`4b4a8838656df8a3a5d8b1b360b7e1c5370b1e848120f98f6706a02531e46af2`. +This establishes the helper's native capability, not physical Pixel, Chrome 152, +iOS, original-byte clipboard fidelity or a fix for Gboard keyboard-tile refusal. +The earlier fixture-repair archive is +`#V#computer_file_copy_9e29aa91aa07485a9054d95ddb85de40`, SHA-256 +`35ed4ddb42f3a96469df16d465a9f230770904c5e402c68d9f34fde3a6ba6cd0`; +it establishes setup recovery only. Both supplied archive checksums were verified. +No public deployment was requested by this task. diff --git a/tests/browser/imagePickerAuthenticated.cjs b/tests/browser/imagePickerAuthenticated.cjs index 80ecab674..dc2b5ed5e 100644 --- a/tests/browser/imagePickerAuthenticated.cjs +++ b/tests/browser/imagePickerAuthenticated.cjs @@ -28,9 +28,9 @@ const save = () => fs.writeFileSync(path.join(output, 'results.json'), JSON.stri await page.route('**/von/generate*', async route => { const body = route.request().postDataJSON(); result.requests.push({ prompt: body.prompt, image_attachment_ids: body.image_attachment_ids, - session_id: body.session_id, window_session: !!route.request().headers()['x-von-window-session'] }); + session_id: body.conversation_session_id, window_session: !!route.request().headers()['x-von-window-session'] }); await route.fulfill({ json: { response: 'Fixture request received.', success: true, - terminal_status: 'completed', session_id: body.session_id, llm_debug: { model: 'fixture-no-model' } } }); + terminal_status: 'completed', session_id: body.conversation_session_id, llm_debug: { model: 'fixture-no-model' } } }); }); await page.goto(`${url}/von/`); const health = await (await context.request.get(`${url}/health`)).json(); @@ -46,7 +46,10 @@ const save = () => fs.writeFileSync(path.join(output, 'results.json'), JSON.stri }).catch(() => null), { timeout: 20000 }).toBe(actor); const created = page.waitForResponse(r => r.url().endsWith('/api/session/create_chat_session') && r.request().method() === 'POST'); await page.getByRole('button', { name: 'New conversation', exact: true }).click(); - assert.equal((await created).status(), 200); + const creation = await created; + assert.equal(creation.status(), 200); + result.created_session_id = (await creation.json()).session_id; + assert(result.created_session_id); const input = page.locator('#promptInput'); await expect(input).toBeEnabled(); await input.fill('Describe this screenshot\nKeep my caption'); @@ -97,6 +100,7 @@ const save = () => fs.writeFileSync(path.join(output, 'results.json'), JSON.stri await page.locator('#sendButton').click(); await expect.poll(() => result.requests.length).toBe(1); assert.deepEqual(result.requests[0].image_attachment_ids, [replacement]); + assert.equal(result.requests[0].session_id, result.created_session_id); assert(!result.requests[0].image_attachment_ids.includes(removed)); assert.equal(result.requests[0].prompt, 'Describe this screenshot\nKeep my caption'); assert(result.requests[0].window_session); @@ -115,7 +119,7 @@ const save = () => fs.writeFileSync(path.join(output, 'results.json'), JSON.stri const retry = page.getByRole('button', { name: 'Retry upload of retry.png', exact: true }); await expect(retry).toBeVisible(); await expect(input).toHaveValue('Preserve this failed-upload draft'); - await page.locator('#sendButton').click(); + await expect(page.locator('#sendButton')).toBeDisabled(); assert.equal(result.requests.length, 2); await page.unroute('**/von/api/images/upload'); const retried = page.waitForResponse(r => r.url().endsWith('/api/images/upload'));