Search conversation topics with title and participant priority

docs/engineering/unified_conversations_and_participant_profiles.md

@@ -56,7 +56,8 @@ losing the read acknowledgement. Existing shared-conversation invitation receipt
 remain authoritative. Direct messages retain canonical `read_by` state. Loading a
 background source alone does not mark it read.
 
-The title/participant and Unread filters apply to loaded catalogue rows. **Load
+The Unread filter applies to loaded catalogue rows; title/participant matching is
+combined with indexed chat-content search as described below. **Load
 older conversations** extends the bounded list. When further pages exist the badge
 is an explicit lower bound (`N+`), not a claimed total over unexamined history.
 Existing loaded pins remain visible across refreshes; discovering a pin beyond the
@@ -150,3 +151,41 @@ reference concept. Its `attributes.message_exchange_reference` and description
 retain the source locator; messages remain canonical message concepts. References
 are stable per actor/exchange, not shared ownership or an access grant. Repeated
 copies recheck source access. No existing message migration is required.
+
+## Title, participant and topic search
+
+The Conversations search field now combines loaded title/participant matches
+(including participant display names and normalised concept IDs) with the existing
+`GET /von/api/session/conversation_search?match_mode=hybrid` retrieval path.
+Relevance ordering puts title/participant matches in a separate first tier;
+content scores cannot displace them. Within each tier scores, activity and session
+ID resolve order. Search bypasses the tray's recency cutoff and pin ordering;
+clearing it restores the normal activity view. API `sort=updated` remains an
+explicit activity-order option. Signed search continuations include the ranking
+version and tier so paging cannot skip semantic-only results after a title match.
+
+Content retrieval queries the existing RAG embeddings of visible user/assistant
+chat contributions, with lexical retrieval alongside it. No embedding model is
+called by the browser and no new index or database migration is introduced.
+System/tool/debug contributions are excluded. Retrieved session IDs must pass the
+existing actor-visible conversation intersection. Person/agent message exchanges
+retain loaded title/participant search; their Vontology message bodies are not
+part of the chat embedding index. All-organisations expands message metadata only;
+chat content search stays in the current window's authorised context.
+
+Typing pauses for 300 ms before a request. A response from an earlier query or
+organisation is discarded. Content is queried once per input, not on each catalogue
+poll. Pages contain at most 100 results, with an explicit load-more action. Existing
+backend bounds remain: 500 accessible conversations and up to 10 owner scopes,
+with a shared 500-candidate retrieval budget. Coverage warnings and retrieval
+failures remain visible; a failed content lookup does not hide metadata matches.
+These limits mean the search is not an exhaustive inventory of all past exchanges.
+
+New chat contributions already use the existing RAG upsert path. Older or failed
+embeddings require the existing operator-controlled chat-history reindex service
+(`reindex_chat_history_for_user_namespace` or chunked session reindex, exposed by
+the admin chat-history reindex route). The lexical backfill utility
+`utilities/backfill_conversation_search_index.py` repairs title/visible-text
+projections only; it does not backfill embeddings. Inspect coverage and use bounded,
+actor/namespace-specific reindexing rather than scanning history on every search.
+This change does not run a live backfill or certify current production index coverage.

src/backend/services/conversation_search_service.py

@@ -27,6 +27,10 @@
 _VALID_MATCH_MODES = frozenset({"lexical", "semantic", "hybrid"})
 _VALID_SORTS = frozenset({"relevance", "updated"})
 _MAX_CANDIDATES = 500
+_RANKING_VERSION = 2
+_METADATA_MATCH_FIELDS = frozenset(
+    {"title", "display_name", "display_name_override", "participant"}
+)
 _SEARCH_CURSOR_TTL_SECONDS = 24 * 60 * 60
 
 
@@ -47,6 +51,11 @@ def _safe_float(value: Any) -> float:
         return 0.0
 
 
+def _metadata_priority(row: Mapping[str, Any]) -> int:
+    fields = (row.get("match") or {}).get("fields", [])
+    return int(bool(_METADATA_MATCH_FIELDS.intersection(fields)))
+
+
 def _row_timestamp(row: Mapping[str, Any]) -> str:
     return str(row.get("last_message_at") or row.get("created_at") or "")
 
@@ -535,6 +544,13 @@ def search_actor_conversations(
         terms = [term.casefold() for term in query_text.split() if term.strip()]
         for session_id, row in accessible_by_session.items():
             display_name = str(row.get("session_name") or "")
+            participants = " ".join(
+                str(value) for value in (row.get("participant_ids") or [])
+            ).casefold()
+            participant_text = participants + " " + participants.replace("_", " ")
+            if terms and all(term in participant_text for term in terms):
+                scores[session_id] = scores.get(session_id, 0.0) + 6.0
+                matched_fields.setdefault(session_id, set()).add("participant")
             if terms and all(term in display_name.casefold() for term in terms):
                 scores[session_id] = scores.get(session_id, 0.0) + 6.0
                 matched_fields.setdefault(session_id, set()).add("display_name")
@@ -620,6 +636,7 @@ def search_actor_conversations(
     else:
         result_rows.sort(
             key=lambda row: (
+                _metadata_priority(row),
                 _safe_float((row.get("match") or {}).get("score")),
                 _row_timestamp(row),
                 str(row.get("session_id") or ""),
@@ -639,6 +656,7 @@ def search_actor_conversations(
         "trashed_only": trashed_only,
         "index_version": chat_history_service.CONVERSATION_SEARCH_INDEX_VERSION,
         "index_generation": index_generation,
+        "ranking_version": _RANKING_VERSION,
     }
     if cursor:
         try:
@@ -670,6 +688,7 @@ def search_actor_conversations(
             ]
         else:
             marker = (
+                int(position.get("metadata_priority", 0)),
                 _safe_float(position.get("score")),
                 str(position.get("timestamp") or ""),
                 str(position.get("session_id") or ""),
@@ -678,6 +697,7 @@ def search_actor_conversations(
                 row
                 for row in result_rows
                 if (
+                    _metadata_priority(row),
                     _safe_float((row.get("match") or {}).get("score")),
                     _row_timestamp(row),
                     str(row.get("session_id") or ""),
@@ -695,6 +715,7 @@ def search_actor_conversations(
             "session_id": str(final_row.get("session_id") or ""),
         }
         if sort_mode == "relevance":
+            position["metadata_priority"] = _metadata_priority(final_row)
             position["score"] = _safe_float((final_row.get("match") or {}).get("score"))
         next_cursor = encode_opaque_cursor(
             purpose="conversation_search",

src/frontend/web/von_interface/static/js/chatTab.js

@@ -3,7 +3,7 @@ import { initialiseCompactChatComposer, isCompactComposer, resizeCompactDraft, s
 import { updateExecutionCost } from './components/executionCost.js';
 import { canUseWorkflowStudio } from './workflowStudioAccess.js';
 import { createChatSteeringControls } from './components/chatSteeringControls.js';
-import { directConversationRows, activeMessageConversationId, showChatConversation, resetConversationCatalogue, renderMessageConversationRow, mountCatalogueControls, initialiseConversationCatalogue, selectMessageConversation, filterCatalogueRows } from './components/conversationCatalogue.js';
+import { directConversationRows, activeMessageConversationId, showChatConversation, resetConversationCatalogue, renderMessageConversationRow, mountCatalogueControls, initialiseConversationCatalogue, selectMessageConversation, filterCatalogueRows, catalogueSearchActive, rankCatalogueSearchRows } from './components/conversationCatalogue.js';
 import { catalogueHasMore } from './components/conversationCatalogue.js';
 import { profileButton, participantAvatar, participantIdentityAvatar } from './components/participantProfile.js';
 import { setButtonLabel } from './utils/buttonLabel.js';
@@ -27357,7 +27357,7 @@ function renderChatSessionTabs(sessions, activeSessionId) {
     const unreadBadge = document.getElementById('conversationUnreadBadge');
     const unreadCount = [...canonicalSessions, ...directConversationRows()].filter(row => !isConversationHidden(row.session_id) && row.shared_unread_count > 0).length;
     if (unreadBadge) { unreadBadge.hidden = !unreadCount; unreadBadge.textContent = `${unreadCount}${catalogueHasMore() ? '+' : ''}`; unreadBadge.setAttribute('aria-label', `${catalogueHasMore() ? 'At least ' : ''}${unreadCount} unread conversations`); }
-    if (canonicalSessions.length === 0 && directConversationRows().length === 0) {
+    if (canonicalSessions.length === 0 && directConversationRows().length === 0 && !catalogueSearchActive()) {
         renderChatSessionTabsPlaceholder('empty');
         mountCatalogueControls(container);
         lastRenderedSessionCount = 0;
@@ -27417,7 +27417,7 @@ function renderChatSessionTabs(sessions, activeSessionId) {
         recentWindowDays: conversationHistorySettings.recentWindowDays,
         showAll: false
     });
-    let visibleSessions = Array.isArray(filteredResult.sessionsToRender)
+    let visibleSessions = catalogueSearchActive() ? sessionsAfterHiddenFilter : Array.isArray(filteredResult.sessionsToRender)
         ? filteredResult.sessionsToRender
         : [];
     const newestAgentSessionId = agentCreatedSessionVisibilityState.newestVisibleSessionId;
@@ -27502,7 +27502,9 @@ function renderChatSessionTabs(sessions, activeSessionId) {
             return !pinnedSessionIdsInFilteredOrder.has(sid);
         })
     ];
-    const orderedVisibleSessions = [...pinnedSessions, ...unpinnedSessions];
+    const orderedVisibleSessions = catalogueSearchActive()
+        ? rankCatalogueSearchRows(visibleSessions)
+        : [...pinnedSessions, ...unpinnedSessions];
 
     container.hidden = false;
     const focusedId = container.contains(document.activeElement) ? document.activeElement?.closest('[data-session-id]')?.dataset.sessionId : null;

src/frontend/web/von_interface/static/js/components/conversationCatalogue.js

@@ -26,8 +26,69 @@ let onChange = () => {};
 let chatReadObserver = null;
 let chatReadPending = false;
 
+let searchRows = [];
+let searchCursor = null;
+let searchGeneration = 0;
+let searchTimer = null;
+let searchStatus = '';
+let searchPending = false;
+
+export function catalogueSearchActive() { return Boolean(filterText); }
+
+function metadataMatch(row) {
+    const text = `${row.session_name || ''} ${(row.participant_ids || []).join(' ')} ${(row.participant_profiles || []).map(profile => profile.display_name || '').join(' ')}`.toLocaleLowerCase();
+    const searchable = `${text} ${text.replaceAll('_', ' ')}`;
+    return filterText.split(/\s+/).every(term => searchable.includes(term))
+        || (row.match?.fields || []).some(field => ['title', 'display_name', 'display_name_override', 'participant'].includes(field));
+}
+
+export function rankCatalogueSearchRows(input) {
+    if (!filterText) return input;
+    return input.slice().sort((a, b) => Number(metadataMatch(b)) - Number(metadataMatch(a))
+        || (Number(b.match?.score) || 0) - (Number(a.match?.score) || 0)
+        || (Date.parse(b.last_message_at) || 0) - (Date.parse(a.last_message_at) || 0)
+        || String(a.session_id).localeCompare(String(b.session_id)));
+}
+
 export function filterCatalogueRows(input) {
-    return input.filter(row => (!unreadOnly || row.shared_unread_count > 0) && (!filterText || `${row.session_name || ''} ${(row.participant_ids || []).join(' ')}`.toLocaleLowerCase().includes(filterText)));
+    const combined = new Map(input.map(row => [row.session_id, row]));
+    if (filterText) searchRows.forEach(row => combined.set(row.session_id, { ...row, ...combined.get(row.session_id), match: row.match }));
+    const matchedIds = new Set(searchRows.map(row => row.session_id));
+    return rankCatalogueSearchRows([...combined.values()].filter(row => (!unreadOnly || row.shared_unread_count > 0)
+        && (!filterText || metadataMatch(row) || matchedIds.has(row.session_id))));
+}
+
+export async function searchCatalogueContent({ append = false } = {}) {
+    if (!filterText || (append && !searchCursor)) return;
+    const expected = ++searchGeneration;
+    const contextGeneration = generation;
+    const org = getSessionScopedOrgId();
+    const query = filterText;
+    const cursor = append ? searchCursor : null;
+    searchPending = true;
+    searchStatus = 'Searching conversation content…';
+    onChange();
+    try {
+        const params = new URLSearchParams({ q: query, match_mode: 'hybrid', sort: 'relevance', page_size: '100' });
+        if (cursor) params.set('cursor', cursor);
+        const data = await getJson(`/von/api/session/conversation_search?${params}`);
+        if (expected !== searchGeneration || contextGeneration !== generation || org !== getSessionScopedOrgId()) return;
+        if (data.success !== true) throw new Error('Search unavailable');
+        searchRows = append ? [...new Map([...searchRows, ...(data.results || [])].map(row => [row.session_id, row])).values()] : data.results || [];
+        searchCursor = data.next_cursor || null;
+        const unavailable = ['unavailable', 'partial_results'].includes(data.retrieval?.semantic?.status);
+        searchStatus = unavailable
+            ? 'Content search is unavailable or incomplete. Title and participant matches remain available.'
+            : data.coverage_complete === false ? 'Searching indexed chat content; some history may not be indexed.' : '';
+    } catch (_) {
+        if (expected !== searchGeneration || contextGeneration !== generation || org !== getSessionScopedOrgId()) return;
+        searchStatus = 'Content search could not finish. Title and participant matches remain available. Retry search.';
+    } finally {
+        if (expected === searchGeneration && contextGeneration === generation && org === getSessionScopedOrgId()) {
+            searchPending = false;
+            onChange();
+        }
+    }
 }
 
 export function directConversationRows() { return rows; }
@@ -43,6 +104,15 @@ export function showChatConversation() {
 
 export function resetConversationCatalogue() {
     generation += 1;
+    searchGeneration += 1;
+    clearTimeout(searchTimer);
+    searchRows = [];
+    searchCursor = null;
+    searchStatus = '';
+    searchPending = false;
+    filterText = '';
+    const searchInput = document.querySelector('.catalogue-filters input[type="search"]');
+    if (searchInput) searchInput.value = '';
     rows = [];
     catalogueRows = [];
     nextCursor = null;
@@ -227,6 +297,19 @@ export function mountCatalogueControls(container) {
         };
         container.append(moreButton);
     }
+    if (filterText) {
+        const status = document.createElement('div');
+        status.className = 'conversation-source-status'; status.setAttribute('role', 'status');
+        status.textContent = searchStatus || 'Title and participant matches first. Content search covers indexed chats in the current organisation.';
+        container.append(status);
+        if (searchCursor || searchStatus.includes('Retry search')) {
+            const next = document.createElement('button'); next.type = 'button';
+            next.textContent = searchCursor ? 'Load more search results' : 'Retry search';
+            next.disabled = searchPending;
+            next.onclick = () => void searchCatalogueContent({ append: Boolean(searchCursor) });
+            container.append(next);
+        }
+    }
     if (sourceError) {
         const status = document.createElement('div');
         status.className = 'conversation-source-status'; status.setAttribute('role', 'status');
@@ -239,12 +322,20 @@ export function initialiseConversationCatalogue({ render, acceptChats, currentCh
     onChange = render;
     acceptChatRows = acceptChats || (() => {});
     const controls = document.createElement('div'); controls.className = 'catalogue-filters';
-    const filter = document.createElement('input'); filter.type = 'search'; filter.placeholder = 'Filter title or participant'; filter.setAttribute('aria-label', 'Filter listed conversations by title or participant');
-    filter.oninput = () => { filterText = filter.value.trim().toLocaleLowerCase(); render(); };
+    const filter = document.createElement('input'); filter.type = 'search'; filter.placeholder = 'Search title, participant or topic'; filter.setAttribute('aria-label', 'Search conversations by title, participant or topic');
+    filterText = '';
+    filter.oninput = () => {
+        filterText = filter.value.trim().toLocaleLowerCase();
+        searchGeneration += 1;
+        clearTimeout(searchTimer);
+        searchRows = []; searchCursor = null; searchStatus = ''; searchPending = false;
+        render();
+        if (filterText) searchTimer = setTimeout(() => void searchCatalogueContent(), 300);
+    };
     const unread = document.createElement('button'); unread.type = 'button'; unread.textContent = 'Unread'; unread.setAttribute('aria-pressed', 'false');
     unread.onclick = () => { unreadOnly = !unreadOnly; unread.setAttribute('aria-pressed', String(unreadOnly)); render(); };
     const contexts = document.createElement('button'); contexts.type = 'button'; contexts.textContent = 'All organisations'; contexts.title = 'Include your message exchanges from other organisations'; contexts.setAttribute('aria-pressed', 'false');
-    contexts.onclick = () => { allContexts = !allContexts; contexts.setAttribute('aria-pressed', String(allContexts)); resetConversationCatalogue(); void refreshMessageCatalogue(); };
+    contexts.onclick = () => { allContexts = !allContexts; contexts.setAttribute('aria-pressed', String(allContexts)); resetConversationCatalogue(); void refreshMessageCatalogue(); void searchCatalogueContent(); };
     controls.append(filter, unread, contexts);
     document.querySelector('.conversation-tray-header')?.after(controls);
     const refresh = async () => {

tests/backend/test_conversation_management_routes.py

@@ -553,3 +553,24 @@ def _lexical_candidates(**kwargs):
         "/von/api/session/conversation_search?q=Search+fixture&page_size=10"
     ).get_json()
     assert target in {row["session_id"] for row in rediscovered["results"]}
+
+
+def test_search_api_returns_topic_only_match_after_title(monkeypatch, app_client):
+    service = von_routes.conversation_search_service
+    monkeypatch.setattr(von_routes, "get_effective_context", _effective_context)
+    monkeypatch.setattr(service, "list_actor_conversations", lambda **kw: {
+        "conversations": [
+            {"session_id": "title", "session_name": "Cooling buildings"},
+            {"session_id": "topic", "session_name": "Design"},
+        ], "coverage_complete": True,
+    })
+    monkeypatch.setattr(service, "_lexical_candidates", lambda **kw: ([], {"status": "valid_empty"}))
+    monkeypatch.setattr(service, "search_conversation_preference_session_ids", lambda **kw: [])
+    monkeypatch.setattr(service, "get_rag_service", lambda: types.SimpleNamespace(query=lambda *args, **kw: [
+        {"score": 100, "text": "Passive ventilation reduces indoor temperatures", "metadata": {"session_id": "topic", "role": "user"}},
+    ]))
+    response = app_client.get("/von/api/session/conversation_search?q=cooling+buildings&match_mode=hybrid")
+    assert response.status_code == 200
+    rows = response.get_json()["results"]
+    assert [row["session_id"] for row in rows] == ["title", "topic"]
+    assert rows[1]["match"]["fields"] == ["semantic_content"]