diff --git a/docs/engineering/external_conversation_import.md b/docs/engineering/external_conversation_import.md index 19728df9..e958ea91 100644 --- a/docs/engineering/external_conversation_import.md +++ b/docs/engineering/external_conversation_import.md @@ -48,6 +48,13 @@ The capability has five layers: 3. Only ordinary user-visible human and assistant text is projected into the actor-scoped chat-history store. The projection carries external actor and event provenance and is marked read-only. + Recorded message dates are preserved. Missing message dates use the source + file's last-modified time, labelled `source_file_modified`; the original + source timestamp remains null and import time is recorded separately. + Re-import repairs older import-time fallbacks only for unchanged events. + Known providers use local Codex, Claude Code, Copilot and Gemini brand assets + in conversation rows and assistant avatars; these do not imply a registered + Von participant identity. 4. Continue creates a new native conversation with a lineage receipt and a copy of the visible history. The source snapshot remains unchanged. 5. Machine-wide discovery and execution use a persisted batch/item controller. diff --git a/src/backend/services/chat_history_service.py b/src/backend/services/chat_history_service.py index af990463..a3345c91 100644 --- a/src/backend/services/chat_history_service.py +++ b/src/backend/services/chat_history_service.py @@ -4853,6 +4853,7 @@ def _event_fingerprint(entry: Mapping[str, Any]) -> str: } divergence: Dict[str, Any] | None = None + corrected_fallback_timestamp_count = 0 appended_history = prepared_history action = "new" if existing is not None: @@ -4920,6 +4921,32 @@ def _event_fingerprint(entry: Mapping[str, Any]) -> str: else: action = "refreshed" + # A missing source timestamp is part of the event fingerprint; its + # display fallback is not. Repair only verified, unchanged events. + if not divergence: + repaired_history = [] + for entry in existing_history: + incoming = incoming_by_id.get(entry.get("external_event_id")) + if ( + incoming is not None + and "external_source_timestamp_utc" in entry + and entry["external_source_timestamp_utc"] is None + and incoming.get("external_timestamp_source") == "source_file_modified" + and _coerce_datetime(incoming.get("timestamp")) is not None + and ( + entry.get("timestamp") != incoming["timestamp"] + or entry.get("external_timestamp_source") != "source_file_modified" + ) + ): + entry = { + **entry, + "timestamp": incoming["timestamp"], + "external_timestamp_source": "source_file_modified", + } + corrected_fallback_timestamp_count += 1 + repaired_history.append(entry) + existing_history = repaired_history + combined_history = [*existing_history, *appended_history] stored_manifest["synchronisation"] = { "schema_version": "external_conversation_synchronisation.v1", @@ -4927,6 +4954,7 @@ def _event_fingerprint(entry: Mapping[str, Any]) -> str: "checked_at_utc": now.isoformat(), "appended_message_count": len(appended_history), "retained_message_count": len(existing_history), + "corrected_fallback_timestamp_count": corrected_fallback_timestamp_count, "divergence": divergence, } search_segments = [ @@ -4974,7 +5002,13 @@ def _event_fingerprint(entry: Mapping[str, Any]) -> str: query[f"{EXTERNAL_CONVERSATION_IMPORT_FIELD}.package_sha256"] = ( existing_manifest.get("package_sha256") ) - if appended_history: + if corrected_fallback_timestamp_count: + # Keep the same package compare-and-swap and avoid conflicting + # $push/$set updates to the history array. + set_fields["history"] = combined_history + set_fields["created_at"] = first_message_at + set_on_insert.pop("created_at", None) + elif appended_history: update["$push"] = {"history": {"$each": appended_history}} try: result = chat_history_coll.update_one( @@ -5041,6 +5075,7 @@ def _event_fingerprint(entry: Mapping[str, Any]) -> str: "action": action, "session_id": session_id, "appended_message_count": len(appended_history), + "corrected_fallback_timestamp_count": corrected_fallback_timestamp_count, "divergence": divergence, "canonical_read_back": { "session_id": session_id, diff --git a/src/backend/services/external_conversation_import_service.py b/src/backend/services/external_conversation_import_service.py index 4f7dfcb5..35d26f21 100644 --- a/src/backend/services/external_conversation_import_service.py +++ b/src/backend/services/external_conversation_import_service.py @@ -30,7 +30,7 @@ EXTERNAL_CONVERSATION_PACKAGE_SCHEMA_VERSION = "external_conversation_package.v1" -EXTERNAL_CONVERSATION_PARSER_VERSION = "2026-09-16.2" +EXTERNAL_CONVERSATION_PARSER_VERSION = "2026-09-16.3" EXTERNAL_CONVERSATION_ORIGIN_KIND = "external_conversation_import" EXTERNAL_CONVERSATION_LINEAGE_SCHEMA_VERSION = "conversation_lineage.v1" MAX_EXTERNAL_CONVERSATION_SOURCE_BYTES = 32 * 1024 * 1024 @@ -1460,6 +1460,8 @@ def parse_external_conversation_file( def _history_projection( package: ExternalConversationPackage, + *, + source_file_modified_at: datetime, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: history: list[dict[str, Any]] = [] total_chars = 0 @@ -1489,7 +1491,10 @@ def _history_projection( { "role": event.source_role, "content": projected, - "timestamp": timestamp or datetime.now(timezone.utc), + "timestamp": timestamp or source_file_modified_at, + "external_timestamp_source": ( + "source_message" if timestamp else "source_file_modified" + ), "external_event_id": event.event_id, "external_event_kind": event.event_kind, "external_source_role": event.source_role, @@ -1621,7 +1626,10 @@ def import_external_conversation_file( source_account=package.source_account, source_workspace=package.source_workspace, ) - history, projection_report = _history_projection(package) + source_file_modified_at = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc) + history, projection_report = _history_projection( + package, source_file_modified_at=source_file_modified_at + ) if not history: raise ExternalConversationImportError( "The source contains no user-visible human or assistant messages.", @@ -1717,6 +1725,7 @@ def import_external_conversation_file( "parser_version": package.parser_version, "source_created_at_utc": package.source_created_at_utc, "source_updated_at_utc": package.source_updated_at_utc, + "source_file_modified_at_utc": source_file_modified_at.isoformat(), "parent_conversation_id": package.parent_conversation_id, "raw_document_concept_id": raw_document_concept_id, "raw_file_copy_concept_id": raw_record_result.get("file_copy_concept_id"), diff --git a/src/frontend/web/von_interface/static/images/providers/OCTICONS-LICENSE b/src/frontend/web/von_interface/static/images/providers/OCTICONS-LICENSE new file mode 100644 index 00000000..aa1fa80e --- /dev/null +++ b/src/frontend/web/von_interface/static/images/providers/OCTICONS-LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 GitHub Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/frontend/web/von_interface/static/images/providers/README.md b/src/frontend/web/von_interface/static/images/providers/README.md new file mode 100644 index 00000000..c220f76e --- /dev/null +++ b/src/frontend/web/von_interface/static/images/providers/README.md @@ -0,0 +1,10 @@ +# Imported conversation provider logos + +Used only to identify the source of imported conversations. Marks belong to their respective owners; they do not indicate endorsement. Assets are served locally. + +Retrieved 16 September 2026: + +- `codex.png`: unmodified Codex app mark bundled with OpenAI VS Code extension `openai.chatgpt-26.908.40401-darwin-arm64`, asset `codex-app-ga-logo-3e5209898ca3.png`. +- `claude.svg`: https://claude.ai/favicon.svg +- `copilot.svg`: https://raw.githubusercontent.com/primer/octicons/main/icons/copilot-24.svg (GitHub Octicons, MIT; see `OCTICONS-LICENSE`). +- `gemini.svg`: https://www.gstatic.com/lamda/images/gemini_sparkle_aurora_33f86dc0c0257da337c63.svg, linked by https://gemini.google.com/. diff --git a/src/frontend/web/von_interface/static/images/providers/claude.svg b/src/frontend/web/von_interface/static/images/providers/claude.svg new file mode 100644 index 00000000..26d103d0 --- /dev/null +++ b/src/frontend/web/von_interface/static/images/providers/claude.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/frontend/web/von_interface/static/images/providers/codex.png b/src/frontend/web/von_interface/static/images/providers/codex.png new file mode 100644 index 00000000..7ef8986e Binary files /dev/null and b/src/frontend/web/von_interface/static/images/providers/codex.png differ diff --git a/src/frontend/web/von_interface/static/images/providers/copilot.svg b/src/frontend/web/von_interface/static/images/providers/copilot.svg new file mode 100644 index 00000000..201ecf40 --- /dev/null +++ b/src/frontend/web/von_interface/static/images/providers/copilot.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/frontend/web/von_interface/static/images/providers/gemini.svg b/src/frontend/web/von_interface/static/images/providers/gemini.svg new file mode 100644 index 00000000..38f56259 --- /dev/null +++ b/src/frontend/web/von_interface/static/images/providers/gemini.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index 729dfd59..ddbbcc6d 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -10,6 +10,7 @@ import { createChatSteeringControls } from './components/chatSteeringControls.js import { directConversationRows, activeMessageConversationId, showChatConversation, resetConversationCatalogue, renderMessageConversationRow, mountCatalogueControls, initialiseConversationCatalogue, selectMessageConversation, filterCatalogueRows, catalogueSearchActive, rankCatalogueSearchRows } from './components/conversationCatalogue.js'; import { catalogueHasMore } from './components/conversationCatalogue.js'; import { profileButton, participantAvatar, participantIdentityAvatar } from './components/participantProfile.js'; +import { importedConversationAvatar } from './components/importedConversationAvatar.js'; import { setButtonLabel } from './utils/buttonLabel.js'; import { createConversationTray } from './components/conversationTray.js'; import { CONVERSATION_LAYOUT_KEY, CONVERSATION_LAYOUT_KEYS, CONVERSATION_NARROW_QUERY, loadConversationLayoutPreferences, normaliseConversationLayout, saveConversationLayoutPreference } from './utils/conversationLayoutPreferences.js'; @@ -27535,7 +27536,9 @@ function renderChatSessionTabs(sessions, activeSessionId) { const header = document.createElement('span'); header.className = 'chat-session-tab-header'; - header.append(participantAvatar({ display_name: session.external_conversation ? displayName : 'Von', avatar_url: session.external_conversation ? null : '/static/VonImageBig.png' })); + header.append(session.external_conversation + ? importedConversationAvatar(session.external_conversation.provider, displayName, { sidebar: true }) + : participantAvatar({ display_name: 'Von', avatar_url: '/static/VonImageBig.png' })); const pinToggle = document.createElement('button'); pinToggle.type = 'button'; @@ -38369,10 +38372,7 @@ function appendMessage(sender, message, turnId, hasLlmDebug = false, isHistory = if (options.contributionTimestamp || timestampStr) messageContainer.dataset.contributionTimestamp = options.contributionTimestamp || timestampStr; let assistantAvatar; if (externalActor) { - assistantAvatar = document.createElement('span'); - assistantAvatar.className = 'chat-imported-actor-avatar'; - assistantAvatar.textContent = String(sender || 'Agent').trim().slice(0, 1).toUpperCase() || 'A'; - assistantAvatar.setAttribute('aria-label', `${sender} (imported actor)`); + assistantAvatar = importedConversationAvatar(externalActor.source_actor_id, sender); } else { assistantAvatar = document.createElement('img'); assistantAvatar.src = '/static/VonImageBig.png'; diff --git a/src/frontend/web/von_interface/static/js/components/importedConversationAvatar.js b/src/frontend/web/von_interface/static/js/components/importedConversationAvatar.js new file mode 100644 index 00000000..302f5abf --- /dev/null +++ b/src/frontend/web/von_interface/static/js/components/importedConversationAvatar.js @@ -0,0 +1,27 @@ +const PROVIDERS = { + codex: { label: 'Codex', file: 'codex.png' }, + claude_code: { label: 'Claude Code', file: 'claude.svg' }, + copilot: { label: 'GitHub Copilot', file: 'copilot.svg' }, + gemini: { label: 'Gemini', file: 'gemini.svg' } +}; + +// Provider branding describes the imported source, not a verified Von identity. +export function importedConversationAvatar(providerOrActor, displayName, { sidebar = false } = {}) { + const provider = String(providerOrActor || '').split(':')[0]; + const brand = Object.hasOwn(PROVIDERS, provider) ? PROVIDERS[provider] : null; + const avatar = document.createElement('span'); + avatar.className = sidebar ? 'participant-avatar imported-provider-avatar' : 'chat-imported-actor-avatar imported-provider-avatar'; + avatar.setAttribute('role', 'img'); + avatar.setAttribute('aria-label', `${brand?.label || displayName || 'Agent'} (imported)`); + avatar.title = `${brand?.label || displayName || 'Agent'} ยท Imported conversation`; + avatar.textContent = String(displayName || brand?.label || 'Agent').trim().slice(0, 1).toUpperCase(); + if (brand) { + const image = document.createElement('img'); + image.src = `/static/images/providers/${brand.file}`; + image.alt = ''; + image.addEventListener('load', () => { avatar.classList.add('has-provider-logo'); }); + image.addEventListener('error', () => { image.remove(); avatar.classList.remove('has-provider-logo'); }); + avatar.append(image); + } + return avatar; +} diff --git a/src/frontend/web/von_interface/static/styles.css b/src/frontend/web/von_interface/static/styles.css index 38260757..0b252ee9 100644 --- a/src/frontend/web/von_interface/static/styles.css +++ b/src/frontend/web/von_interface/static/styles.css @@ -8415,6 +8415,27 @@ body[data-active-tab="settingsTab"] .tab-content-area { font-weight: 700; } +.imported-provider-avatar { + position: relative; + overflow: hidden; +} + +.imported-provider-avatar.has-provider-logo { + color: transparent; + background: #fff; +} + +.imported-provider-avatar img { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + padding: 4px; + box-sizing: border-box; + object-fit: contain; + background: #fff; +} + .external-conversation-bulk-import { display: flex; align-items: center; diff --git a/tests/backend/test_external_conversation_timestamp_fallback.py b/tests/backend/test_external_conversation_timestamp_fallback.py new file mode 100644 index 00000000..65b53b6e --- /dev/null +++ b/tests/backend/test_external_conversation_timestamp_fallback.py @@ -0,0 +1,197 @@ +from datetime import datetime, timezone +import json +import os + +import mongomock + +from src.backend.services import chat_history_service as history_service +from src.backend.services import external_conversation_import_service as importer + + +def test_file_mtime_fills_only_missing_message_dates(monkeypatch, tmp_path): + path = tmp_path / "source.jsonl" + path.write_text( + "\n".join( + json.dumps(record) + for record in [ + {"type": "session_meta", "payload": {"id": "dated-source"}}, + { + "type": "response_item", + "timestamp": "2026-01-02T03:04:05Z", + "payload": {"type": "message", "role": "user", "content": "Dated"}, + }, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": "Undated", + }, + }, + ] + ) + ) + modified = datetime(2026, 2, 3, 4, 5, 6, tzinfo=timezone.utc) + os.utime(path, (modified.timestamp(), modified.timestamp())) + original = importer._history_projection + captured = [] + + def project(package, **kwargs): + result = original(package, **kwargs) + captured.extend(result[0]) + return result + + monkeypatch.setattr(importer, "_history_projection", project) + monkeypatch.setattr( + history_service, + "classify_external_conversation_projection", + lambda **kwargs: "new", + ) + importer.import_external_conversation_file( + path=path, + custodian_user_id="#V#user", + namespace=None, + organisation_concept_id=None, + role_in_org=None, + dry_run=True, + ) + assert captured[0]["timestamp"] == "2026-01-02T03:04:05+00:00" + assert captured[0]["external_timestamp_source"] == "source_message" + assert captured[1]["timestamp"] == modified + assert captured[1]["external_source_timestamp_utc"] is None + assert captured[1]["external_timestamp_source"] == "source_file_modified" + + +def test_reimport_repairs_fallbacks_preserving_source_dates_and_appends(monkeypatch): + collection = mongomock.MongoClient().test.chat_history + monkeypatch.setattr( + history_service, + "get_chat_history_collection_service", + lambda **kwargs: collection, + ) + monkeypatch.setattr( + history_service, + "_read_find_one", + lambda coll, query, projection=None, **kwargs: coll.find_one(query, projection), + ) + manifest = dict( + schema_version="external_conversation_import.v1", + read_only=True, + custodian_user_id="#V#user", + source_identity_key="same", + source_sha256="a" * 64, + package_sha256="b" * 64, + parser_version="old", + provider="codex", + ) + dated = dict( + role="user", + content="Dated", + timestamp="2026-01-02T00:00:00Z", + external_source_timestamp_utc="2026-01-02T00:00:00Z", + external_event_id="one", + ) + undated = dict( + role="assistant", + content="Undated", + timestamp="2026-09-16T00:00:00Z", + external_source_timestamp_utc=None, + external_event_id="two", + ) + + def write(entries, revision): + return history_service.upsert_external_conversation_projection( + user_id="#V#user", + session_id="external", + session_name="Imported", + namespace=None, + organisation_concept_id=None, + role_in_org=None, + history=entries, + manifest={**manifest, "parser_version": revision}, + ) + + write([dated, undated], "old") + repaired = { + **undated, + "timestamp": "2026-01-03T00:00:00Z", + "external_timestamp_source": "source_file_modified", + } + extra = {**dated, "content": "Appended", "external_event_id": "three"} + result = write([dated, repaired, extra], "new") + assert result["corrected_fallback_timestamp_count"] == 1 + assert result["appended_message_count"] == 1 + stored = collection.find_one({}) + assert [e["timestamp"] for e in stored["history"]] == [ + dated["timestamp"], + repaired["timestamp"], + extra["timestamp"], + ] + assert stored["updated_at"] == datetime(2026, 1, 3) + assert write([dated, repaired, extra], "new")["action"] == "unchanged" + # Changed historical content remains a divergence, never a timestamp repair. + divergent = write( + [ + dated, + {**repaired, "content": "Changed", "timestamp": "2026-03-03T00:00:00Z"}, + extra, + ], + "newer", + ) + assert divergent["action"] == "diverged" + assert divergent["corrected_fallback_timestamp_count"] == 0 + assert collection.find_one({})["history"] == stored["history"] + + +def test_repair_corrects_conversation_creation_date(monkeypatch): + collection = mongomock.MongoClient().test.chat_history + monkeypatch.setattr( + history_service, + "get_chat_history_collection_service", + lambda **kwargs: collection, + ) + monkeypatch.setattr( + history_service, + "_read_find_one", + lambda coll, query, projection=None, **kwargs: coll.find_one(query, projection), + ) + kwargs = dict( + user_id="#V#user", + session_id="external", + session_name="Imported", + namespace=None, + organisation_concept_id=None, + role_in_org=None, + ) + manifest = dict( + schema_version="external_conversation_import.v1", + read_only=True, + custodian_user_id="#V#user", + source_identity_key="same", + source_sha256="a" * 64, + package_sha256="b" * 64, + parser_version="old", + provider="codex", + ) + entry = dict( + role="user", + content="Undated", + timestamp="2026-09-16T00:00:00Z", + external_source_timestamp_utc=None, + external_event_id="one", + ) + history_service.upsert_external_conversation_projection( + **kwargs, history=[entry], manifest=manifest + ) + history_service.upsert_external_conversation_projection( + **kwargs, + history=[ + { + **entry, + "timestamp": "2026-01-03T00:00:00Z", + "external_timestamp_source": "source_file_modified", + } + ], + manifest={**manifest, "parser_version": "new"}, + ) + assert collection.find_one({})["created_at"] == datetime(2026, 1, 3) diff --git a/tests/browser/importedConversationAvatars.cjs b/tests/browser/importedConversationAvatars.cjs new file mode 100644 index 00000000..d51ad158 --- /dev/null +++ b/tests/browser/importedConversationAvatars.cjs @@ -0,0 +1,45 @@ +// Exercise the production avatar module, styles and real local brand assets. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); +const { chromium } = require('@playwright/test'); +const root = path.resolve(__dirname, '../../src/frontend/web/von_interface'); +const evidence = process.argv[2]; +const server = http.createServer((req, res) => { + if (req.url === '/') { + res.setHeader('Content-Type', 'text/html'); + return res.end('

Imported conversations

'); + } + const file = path.resolve(root, `.${req.url}`); + if (!file.startsWith(`${root}/static/`) || !fs.existsSync(file)) return res.writeHead(404).end(); + const type = { '.js': 'text/javascript', '.css': 'text/css', '.svg': 'image/svg+xml', '.png': 'image/png' }[path.extname(file)]; + res.setHeader('Content-Type', type || 'text/plain'); + res.end(fs.readFileSync(file)); +}); +(async () => { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ viewport: { width: 760, height: 600 } }); + await page.goto(`http://127.0.0.1:${server.address().port}`); + await page.evaluate(async () => { + const { importedConversationAvatar: avatar } = await import('/static/js/components/importedConversationAvatar.js'); + for (const [provider, label] of [['codex', 'Codex'], ['claude_code', 'Claude Code'], ['copilot', 'GitHub Copilot'], ['gemini', 'Gemini'], ['unknown', 'Other agent']]) { + const row = document.createElement('div'); + row.style.cssText = 'display:flex;align-items:center;gap:16px;margin:20px 0'; + row.append(avatar(provider, label, { sidebar: true }), avatar(`${provider}:agent`, label), document.createTextNode(label)); + document.querySelector('main').append(row); + } + }); + await page.waitForFunction(() => [...document.images].every(image => image.complete && image.naturalWidth > 0)); + assert.equal(await page.locator('.has-provider-logo').count(), 8); + assert.equal(await page.locator('[aria-label="Other agent (imported)"] img').count(), 0); + assert.equal(await page.locator('.participant-avatar').first().evaluate(el => el.getBoundingClientRect().width), 32); + assert.equal(await page.locator('.chat-imported-actor-avatar').first().evaluate(el => el.getBoundingClientRect().width), 40); + if (evidence) { fs.mkdirSync(evidence, { recursive: true }); await page.screenshot({ path: path.join(evidence, 'imported-provider-logos.png') }); } + await page.locator('img').first().evaluate(image => image.dispatchEvent(new Event('error'))); + assert.equal(await page.locator('.participant-avatar').first().innerText(), 'C'); + console.log('Provider logos loaded at sidebar/message sizes; unknown and failed-image fallbacks verified.'); + } finally { await browser.close(); server.close(); } +})().catch(error => { console.error(error); server.close(); process.exitCode = 1; });