diff --git a/docs/engineering/unified_conversations_and_participant_profiles.md b/docs/engineering/unified_conversations_and_participant_profiles.md index 3be94e69..4a8921e1 100644 --- a/docs/engineering/unified_conversations_and_participant_profiles.md +++ b/docs/engineering/unified_conversations_and_participant_profiles.md @@ -56,7 +56,8 @@ losing the read acknowledgement. Existing shared-conversation invitation receipt remain authoritative. Direct messages retain canonical `read_by` state. Loading a background source alone does not mark it read. -The title/participant and Unread filters apply to loaded catalogue rows. **Load +The Unread filter applies to loaded catalogue rows; title/participant matching is +combined with indexed chat-content search as described below. **Load older conversations** extends the bounded list. When further pages exist the badge is an explicit lower bound (`N+`), not a claimed total over unexamined history. Existing loaded pins remain visible across refreshes; discovering a pin beyond the @@ -150,3 +151,41 @@ reference concept. Its `attributes.message_exchange_reference` and description retain the source locator; messages remain canonical message concepts. References are stable per actor/exchange, not shared ownership or an access grant. Repeated copies recheck source access. No existing message migration is required. + +## Title, participant and topic search + +The Conversations search field now combines loaded title/participant matches +(including participant display names and normalised concept IDs) with the existing +`GET /von/api/session/conversation_search?match_mode=hybrid` retrieval path. +Relevance ordering puts title/participant matches in a separate first tier; +content scores cannot displace them. Within each tier scores, activity and session +ID resolve order. Search bypasses the tray's recency cutoff and pin ordering; +clearing it restores the normal activity view. API `sort=updated` remains an +explicit activity-order option. Signed search continuations include the ranking +version and tier so paging cannot skip semantic-only results after a title match. + +Content retrieval queries the existing RAG embeddings of visible user/assistant +chat contributions, with lexical retrieval alongside it. No embedding model is +called by the browser and no new index or database migration is introduced. +System/tool/debug contributions are excluded. Retrieved session IDs must pass the +existing actor-visible conversation intersection. Person/agent message exchanges +retain loaded title/participant search; their Vontology message bodies are not +part of the chat embedding index. All-organisations expands message metadata only; +chat content search stays in the current window's authorised context. + +Typing pauses for 300 ms before a request. A response from an earlier query or +organisation is discarded. Content is queried once per input, not on each catalogue +poll. Pages contain at most 100 results, with an explicit load-more action. Existing +backend bounds remain: 500 accessible conversations and up to 10 owner scopes, +with a shared 500-candidate retrieval budget. Coverage warnings and retrieval +failures remain visible; a failed content lookup does not hide metadata matches. +These limits mean the search is not an exhaustive inventory of all past exchanges. + +New chat contributions already use the existing RAG upsert path. Older or failed +embeddings require the existing operator-controlled chat-history reindex service +(`reindex_chat_history_for_user_namespace` or chunked session reindex, exposed by +the admin chat-history reindex route). The lexical backfill utility +`utilities/backfill_conversation_search_index.py` repairs title/visible-text +projections only; it does not backfill embeddings. Inspect coverage and use bounded, +actor/namespace-specific reindexing rather than scanning history on every search. +This change does not run a live backfill or certify current production index coverage. diff --git a/src/backend/services/conversation_search_service.py b/src/backend/services/conversation_search_service.py index 6ff33a2d..00f5f9d7 100644 --- a/src/backend/services/conversation_search_service.py +++ b/src/backend/services/conversation_search_service.py @@ -27,6 +27,10 @@ _VALID_MATCH_MODES = frozenset({"lexical", "semantic", "hybrid"}) _VALID_SORTS = frozenset({"relevance", "updated"}) _MAX_CANDIDATES = 500 +_RANKING_VERSION = 2 +_METADATA_MATCH_FIELDS = frozenset( + {"title", "display_name", "display_name_override", "participant"} +) _SEARCH_CURSOR_TTL_SECONDS = 24 * 60 * 60 @@ -47,6 +51,11 @@ def _safe_float(value: Any) -> float: return 0.0 +def _metadata_priority(row: Mapping[str, Any]) -> int: + fields = (row.get("match") or {}).get("fields", []) + return int(bool(_METADATA_MATCH_FIELDS.intersection(fields))) + + def _row_timestamp(row: Mapping[str, Any]) -> str: return str(row.get("last_message_at") or row.get("created_at") or "") @@ -535,6 +544,13 @@ def search_actor_conversations( terms = [term.casefold() for term in query_text.split() if term.strip()] for session_id, row in accessible_by_session.items(): display_name = str(row.get("session_name") or "") + participants = " ".join( + str(value) for value in (row.get("participant_ids") or []) + ).casefold() + participant_text = participants + " " + participants.replace("_", " ") + if terms and all(term in participant_text for term in terms): + scores[session_id] = scores.get(session_id, 0.0) + 6.0 + matched_fields.setdefault(session_id, set()).add("participant") if terms and all(term in display_name.casefold() for term in terms): scores[session_id] = scores.get(session_id, 0.0) + 6.0 matched_fields.setdefault(session_id, set()).add("display_name") @@ -620,6 +636,7 @@ def search_actor_conversations( else: result_rows.sort( key=lambda row: ( + _metadata_priority(row), _safe_float((row.get("match") or {}).get("score")), _row_timestamp(row), str(row.get("session_id") or ""), @@ -639,6 +656,7 @@ def search_actor_conversations( "trashed_only": trashed_only, "index_version": chat_history_service.CONVERSATION_SEARCH_INDEX_VERSION, "index_generation": index_generation, + "ranking_version": _RANKING_VERSION, } if cursor: try: @@ -670,6 +688,7 @@ def search_actor_conversations( ] else: marker = ( + int(position.get("metadata_priority", 0)), _safe_float(position.get("score")), str(position.get("timestamp") or ""), str(position.get("session_id") or ""), @@ -678,6 +697,7 @@ def search_actor_conversations( row for row in result_rows if ( + _metadata_priority(row), _safe_float((row.get("match") or {}).get("score")), _row_timestamp(row), str(row.get("session_id") or ""), @@ -695,6 +715,7 @@ def search_actor_conversations( "session_id": str(final_row.get("session_id") or ""), } if sort_mode == "relevance": + position["metadata_priority"] = _metadata_priority(final_row) position["score"] = _safe_float((final_row.get("match") or {}).get("score")) next_cursor = encode_opaque_cursor( purpose="conversation_search", diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index 55c46bae..e4381f6b 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -3,7 +3,7 @@ import { initialiseCompactChatComposer, isCompactComposer, resizeCompactDraft, s import { updateExecutionCost } from './components/executionCost.js'; import { canUseWorkflowStudio } from './workflowStudioAccess.js'; import { createChatSteeringControls } from './components/chatSteeringControls.js'; -import { directConversationRows, activeMessageConversationId, showChatConversation, resetConversationCatalogue, renderMessageConversationRow, mountCatalogueControls, initialiseConversationCatalogue, selectMessageConversation, filterCatalogueRows } from './components/conversationCatalogue.js'; +import { directConversationRows, activeMessageConversationId, showChatConversation, resetConversationCatalogue, renderMessageConversationRow, mountCatalogueControls, initialiseConversationCatalogue, selectMessageConversation, filterCatalogueRows, catalogueSearchActive, rankCatalogueSearchRows } from './components/conversationCatalogue.js'; import { catalogueHasMore } from './components/conversationCatalogue.js'; import { profileButton, participantAvatar, participantIdentityAvatar } from './components/participantProfile.js'; import { setButtonLabel } from './utils/buttonLabel.js'; @@ -27357,7 +27357,7 @@ function renderChatSessionTabs(sessions, activeSessionId) { const unreadBadge = document.getElementById('conversationUnreadBadge'); const unreadCount = [...canonicalSessions, ...directConversationRows()].filter(row => !isConversationHidden(row.session_id) && row.shared_unread_count > 0).length; if (unreadBadge) { unreadBadge.hidden = !unreadCount; unreadBadge.textContent = `${unreadCount}${catalogueHasMore() ? '+' : ''}`; unreadBadge.setAttribute('aria-label', `${catalogueHasMore() ? 'At least ' : ''}${unreadCount} unread conversations`); } - if (canonicalSessions.length === 0 && directConversationRows().length === 0) { + if (canonicalSessions.length === 0 && directConversationRows().length === 0 && !catalogueSearchActive()) { renderChatSessionTabsPlaceholder('empty'); mountCatalogueControls(container); lastRenderedSessionCount = 0; @@ -27417,7 +27417,7 @@ function renderChatSessionTabs(sessions, activeSessionId) { recentWindowDays: conversationHistorySettings.recentWindowDays, showAll: false }); - let visibleSessions = Array.isArray(filteredResult.sessionsToRender) + let visibleSessions = catalogueSearchActive() ? sessionsAfterHiddenFilter : Array.isArray(filteredResult.sessionsToRender) ? filteredResult.sessionsToRender : []; const newestAgentSessionId = agentCreatedSessionVisibilityState.newestVisibleSessionId; @@ -27502,7 +27502,9 @@ function renderChatSessionTabs(sessions, activeSessionId) { return !pinnedSessionIdsInFilteredOrder.has(sid); }) ]; - const orderedVisibleSessions = [...pinnedSessions, ...unpinnedSessions]; + const orderedVisibleSessions = catalogueSearchActive() + ? rankCatalogueSearchRows(visibleSessions) + : [...pinnedSessions, ...unpinnedSessions]; container.hidden = false; const focusedId = container.contains(document.activeElement) ? document.activeElement?.closest('[data-session-id]')?.dataset.sessionId : null; diff --git a/src/frontend/web/von_interface/static/js/components/conversationCatalogue.js b/src/frontend/web/von_interface/static/js/components/conversationCatalogue.js index d5d14b90..0c342111 100644 --- a/src/frontend/web/von_interface/static/js/components/conversationCatalogue.js +++ b/src/frontend/web/von_interface/static/js/components/conversationCatalogue.js @@ -26,8 +26,69 @@ let onChange = () => {}; let chatReadObserver = null; let chatReadPending = false; +let searchRows = []; +let searchCursor = null; +let searchGeneration = 0; +let searchTimer = null; +let searchStatus = ''; +let searchPending = false; + +export function catalogueSearchActive() { return Boolean(filterText); } + +function metadataMatch(row) { + const text = `${row.session_name || ''} ${(row.participant_ids || []).join(' ')} ${(row.participant_profiles || []).map(profile => profile.display_name || '').join(' ')}`.toLocaleLowerCase(); + const searchable = `${text} ${text.replaceAll('_', ' ')}`; + return filterText.split(/\s+/).every(term => searchable.includes(term)) + || (row.match?.fields || []).some(field => ['title', 'display_name', 'display_name_override', 'participant'].includes(field)); +} + +export function rankCatalogueSearchRows(input) { + if (!filterText) return input; + return input.slice().sort((a, b) => Number(metadataMatch(b)) - Number(metadataMatch(a)) + || (Number(b.match?.score) || 0) - (Number(a.match?.score) || 0) + || (Date.parse(b.last_message_at) || 0) - (Date.parse(a.last_message_at) || 0) + || String(a.session_id).localeCompare(String(b.session_id))); +} + export function filterCatalogueRows(input) { - return input.filter(row => (!unreadOnly || row.shared_unread_count > 0) && (!filterText || `${row.session_name || ''} ${(row.participant_ids || []).join(' ')}`.toLocaleLowerCase().includes(filterText))); + const combined = new Map(input.map(row => [row.session_id, row])); + if (filterText) searchRows.forEach(row => combined.set(row.session_id, { ...row, ...combined.get(row.session_id), match: row.match })); + const matchedIds = new Set(searchRows.map(row => row.session_id)); + return rankCatalogueSearchRows([...combined.values()].filter(row => (!unreadOnly || row.shared_unread_count > 0) + && (!filterText || metadataMatch(row) || matchedIds.has(row.session_id)))); +} + +export async function searchCatalogueContent({ append = false } = {}) { + if (!filterText || (append && !searchCursor)) return; + const expected = ++searchGeneration; + const contextGeneration = generation; + const org = getSessionScopedOrgId(); + const query = filterText; + const cursor = append ? searchCursor : null; + searchPending = true; + searchStatus = 'Searching conversation content…'; + onChange(); + try { + const params = new URLSearchParams({ q: query, match_mode: 'hybrid', sort: 'relevance', page_size: '100' }); + if (cursor) params.set('cursor', cursor); + const data = await getJson(`/von/api/session/conversation_search?${params}`); + if (expected !== searchGeneration || contextGeneration !== generation || org !== getSessionScopedOrgId()) return; + if (data.success !== true) throw new Error('Search unavailable'); + searchRows = append ? [...new Map([...searchRows, ...(data.results || [])].map(row => [row.session_id, row])).values()] : data.results || []; + searchCursor = data.next_cursor || null; + const unavailable = ['unavailable', 'partial_results'].includes(data.retrieval?.semantic?.status); + searchStatus = unavailable + ? 'Content search is unavailable or incomplete. Title and participant matches remain available.' + : data.coverage_complete === false ? 'Searching indexed chat content; some history may not be indexed.' : ''; + } catch (_) { + if (expected !== searchGeneration || contextGeneration !== generation || org !== getSessionScopedOrgId()) return; + searchStatus = 'Content search could not finish. Title and participant matches remain available. Retry search.'; + } finally { + if (expected === searchGeneration && contextGeneration === generation && org === getSessionScopedOrgId()) { + searchPending = false; + onChange(); + } + } } export function directConversationRows() { return rows; } @@ -43,6 +104,15 @@ export function showChatConversation() { export function resetConversationCatalogue() { generation += 1; + searchGeneration += 1; + clearTimeout(searchTimer); + searchRows = []; + searchCursor = null; + searchStatus = ''; + searchPending = false; + filterText = ''; + const searchInput = document.querySelector('.catalogue-filters input[type="search"]'); + if (searchInput) searchInput.value = ''; rows = []; catalogueRows = []; nextCursor = null; @@ -227,6 +297,19 @@ export function mountCatalogueControls(container) { }; container.append(moreButton); } + if (filterText) { + const status = document.createElement('div'); + status.className = 'conversation-source-status'; status.setAttribute('role', 'status'); + status.textContent = searchStatus || 'Title and participant matches first. Content search covers indexed chats in the current organisation.'; + container.append(status); + if (searchCursor || searchStatus.includes('Retry search')) { + const next = document.createElement('button'); next.type = 'button'; + next.textContent = searchCursor ? 'Load more search results' : 'Retry search'; + next.disabled = searchPending; + next.onclick = () => void searchCatalogueContent({ append: Boolean(searchCursor) }); + container.append(next); + } + } if (sourceError) { const status = document.createElement('div'); status.className = 'conversation-source-status'; status.setAttribute('role', 'status'); @@ -239,12 +322,20 @@ export function initialiseConversationCatalogue({ render, acceptChats, currentCh onChange = render; acceptChatRows = acceptChats || (() => {}); const controls = document.createElement('div'); controls.className = 'catalogue-filters'; - const filter = document.createElement('input'); filter.type = 'search'; filter.placeholder = 'Filter title or participant'; filter.setAttribute('aria-label', 'Filter listed conversations by title or participant'); - filter.oninput = () => { filterText = filter.value.trim().toLocaleLowerCase(); render(); }; + const filter = document.createElement('input'); filter.type = 'search'; filter.placeholder = 'Search title, participant or topic'; filter.setAttribute('aria-label', 'Search conversations by title, participant or topic'); + filterText = ''; + filter.oninput = () => { + filterText = filter.value.trim().toLocaleLowerCase(); + searchGeneration += 1; + clearTimeout(searchTimer); + searchRows = []; searchCursor = null; searchStatus = ''; searchPending = false; + render(); + if (filterText) searchTimer = setTimeout(() => void searchCatalogueContent(), 300); + }; const unread = document.createElement('button'); unread.type = 'button'; unread.textContent = 'Unread'; unread.setAttribute('aria-pressed', 'false'); unread.onclick = () => { unreadOnly = !unreadOnly; unread.setAttribute('aria-pressed', String(unreadOnly)); render(); }; const contexts = document.createElement('button'); contexts.type = 'button'; contexts.textContent = 'All organisations'; contexts.title = 'Include your message exchanges from other organisations'; contexts.setAttribute('aria-pressed', 'false'); - contexts.onclick = () => { allContexts = !allContexts; contexts.setAttribute('aria-pressed', String(allContexts)); resetConversationCatalogue(); void refreshMessageCatalogue(); }; + contexts.onclick = () => { allContexts = !allContexts; contexts.setAttribute('aria-pressed', String(allContexts)); resetConversationCatalogue(); void refreshMessageCatalogue(); void searchCatalogueContent(); }; controls.append(filter, unread, contexts); document.querySelector('.conversation-tray-header')?.after(controls); const refresh = async () => { diff --git a/tests/backend/test_conversation_management_routes.py b/tests/backend/test_conversation_management_routes.py index d917820b..f9bf38ca 100644 --- a/tests/backend/test_conversation_management_routes.py +++ b/tests/backend/test_conversation_management_routes.py @@ -553,3 +553,24 @@ def _lexical_candidates(**kwargs): "/von/api/session/conversation_search?q=Search+fixture&page_size=10" ).get_json() assert target in {row["session_id"] for row in rediscovered["results"]} + + +def test_search_api_returns_topic_only_match_after_title(monkeypatch, app_client): + service = von_routes.conversation_search_service + monkeypatch.setattr(von_routes, "get_effective_context", _effective_context) + monkeypatch.setattr(service, "list_actor_conversations", lambda **kw: { + "conversations": [ + {"session_id": "title", "session_name": "Cooling buildings"}, + {"session_id": "topic", "session_name": "Design"}, + ], "coverage_complete": True, + }) + monkeypatch.setattr(service, "_lexical_candidates", lambda **kw: ([], {"status": "valid_empty"})) + monkeypatch.setattr(service, "search_conversation_preference_session_ids", lambda **kw: []) + monkeypatch.setattr(service, "get_rag_service", lambda: types.SimpleNamespace(query=lambda *args, **kw: [ + {"score": 100, "text": "Passive ventilation reduces indoor temperatures", "metadata": {"session_id": "topic", "role": "user"}}, + ])) + response = app_client.get("/von/api/session/conversation_search?q=cooling+buildings&match_mode=hybrid") + assert response.status_code == 200 + rows = response.get_json()["results"] + assert [row["session_id"] for row in rows] == ["title", "topic"] + assert rows[1]["match"]["fields"] == ["semantic_content"] diff --git a/tests/backend/test_conversation_search_service.py b/tests/backend/test_conversation_search_service.py index 83c78333..8b881e89 100644 --- a/tests/backend/test_conversation_search_service.py +++ b/tests/backend/test_conversation_search_service.py @@ -595,3 +595,46 @@ def update_one(self, query, update): ] == [1, 3] assert "private system text" not in str(projection) assert "private tool output" not in str(projection) + + +@pytest.mark.parametrize("sort", ["relevance", "updated"]) +def test_metadata_precedes_high_semantic_scores_across_pages(monkeypatch, sort): + from src.backend.services import conversation_search_service as service + + rows = [ + {"session_id": "title", "session_name": "Cooling buildings", "last_message_at": "2020-01-01"}, + {"session_id": "participant", "session_name": "Notes", "participant_ids": ["#V#cooling_buildings_team"], "last_message_at": "2020-01-02"}, + {"session_id": "topic", "session_name": "Design", "last_message_at": "2026-01-01"}, + ] + monkeypatch.setattr(service, "list_actor_conversations", lambda **kw: {"conversations": rows, "coverage_complete": True}) + monkeypatch.setattr(service, "_lexical_candidates", lambda **kw: ([], {"status": "valid_empty"})) + monkeypatch.setattr(service, "search_conversation_preference_session_ids", lambda **kw: []) + calls = [] + + class Rag: + def query(self, query, **kwargs): + calls.append((query, kwargs)) + # Vector results need not contain any query words; neither the + # title nor transcript here shares words with the topic query. + return [ + {"score": 1000, "text": "Passive ventilation reduces indoor temperatures.", "metadata": {"session_id": "topic", "role": "assistant"}}, + {"score": .8, "text": "Shaded facades reduce heat gain.", "metadata": {"session_id": "title", "role": "user"}}, + {"score": 9000, "text": "Private", "metadata": {"session_id": "foreign", "role": "user"}}, + ] + + monkeypatch.setattr(service, "get_rag_service", lambda: Rag()) + found, cursor = [], None + while True: + result = service.search_actor_conversations(actor_user_id="#V#alice", namespace="#V#alice@lab", query="cooling buildings", sort=sort, page_size=1, cursor=cursor) + found.extend(result["results"]) + cursor = result["next_cursor"] + if not cursor: + break + assert [r["session_id"] for r in found] == (["title", "participant", "topic"] if sort == "relevance" else ["topic", "participant", "title"]) + title = next(r for r in found if r["session_id"] == "title") + assert {"display_name", "semantic_content"} <= set(title["match"]["fields"]) + topic = next(r for r in found if r["session_id"] == "topic") + assert topic["match"]["fields"] == ["semantic_content"] + assert "ventilation" in topic["match"]["snippet"] + assert calls[0][1]["namespace"] == "#V#alice@lab" + assert calls[0][1]["permissions_context"]["mode"] == "chat" diff --git a/tests/browser/conversationTopicSearch.cjs b/tests/browser/conversationTopicSearch.cjs new file mode 100644 index 00000000..47738a2d --- /dev/null +++ b/tests/browser/conversationTopicSearch.cjs @@ -0,0 +1,72 @@ +// Candidate catalogue module with synthetic API replies; no login or live data. +// PLAYWRIGHT_BROWSERS_PATH=/tmp/von-playwright node tests/browser/conversationTopicSearch.cjs [evidence-dir] +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); +const { chromium, expect } = require('@playwright/test'); +const root = path.resolve(__dirname, '../../src/frontend/web/von_interface'); +const evidence = process.argv[2]; +const server = http.createServer((req, res) => { + if (req.url === '/') { + res.setHeader('Content-Type', 'text/html'); + return res.end('