From 0ff75a7d5f6d38fd0626b5813eb6e03d5354bf65 Mon Sep 17 00:00:00 2001 From: witbrock Date: Tue, 28 Jul 2026 23:09:41 +0100 Subject: [PATCH 1/2] Implement actor-scoped assertions for visible concepts --- src/backend/db/mongo_client.py | 64 ++- .../integrations/internal_mcp/catalogue.py | 487 +++++++++++------- src/backend/services/adaptive_turn_service.py | 266 +++------- .../services/concept_relation_service.py | 166 ++++-- .../rag_text_relation_sync_service.py | 65 ++- .../services/scoped_assertion_service.py | 381 ++++++++++++++ src/backend/services/text_value_service.py | 131 +++-- .../test_internal_mcp_catalogue_builds.py | 26 +- .../test_mongo_collection_index_guards.py | 42 +- .../test_rag_text_relation_sync_service.py | 44 +- .../backend/test_scoped_assertion_service.py | 248 +++++++++ 11 files changed, 1438 insertions(+), 482 deletions(-) create mode 100644 src/backend/services/scoped_assertion_service.py create mode 100644 tests/backend/test_scoped_assertion_service.py diff --git a/src/backend/db/mongo_client.py b/src/backend/db/mongo_client.py index 95f2a0c63..9f26e7abb 100644 --- a/src/backend/db/mongo_client.py +++ b/src/backend/db/mongo_client.py @@ -514,6 +514,7 @@ def assert_destructive_db_operation_allowed(operation: str) -> None: TEXT_RELATIONS_COLLECTION_NAME = ( "text_relations" # New collection linking concepts to text values ) +SCOPED_KNOWLEDGE_ASSERTIONS_COLLECTION_NAME = "scoped_knowledge_assertions" META_RELATIONS_COLLECTION_NAME = "meta_relations" # New collection for relation elicitation meta-data (JVNAUTOSCI-371) RELATIONSHIP_EXTENT_INDEX_COLLECTION_NAME = "relationship_extent_index" @@ -644,7 +645,9 @@ def _try_preferred_dns_fallback_first() -> bool: _using_fallback_real = True return True except Exception as exc: - logger.warning("[mongo_fallback] Preferred DNS fallback connection failed: %s", exc) + logger.warning( + "[mongo_fallback] Preferred DNS fallback connection failed: %s", exc + ) _mongo_client_real = None _clear_dns_fallback_preference() return False @@ -790,9 +793,7 @@ def get_db() -> Database | None: and MONGO_DNS_FALLBACK_URI and fallback_uri == MONGO_DNS_FALLBACK_URI ): - _mark_dns_fallback_preferred( - "primary connection failure" - ) + _mark_dns_fallback_preferred("primary connection failure") _effective_uri_real = fallback_uri _using_fallback_real = True _last_auto_recovery_check_at = time.monotonic() @@ -1188,6 +1189,40 @@ def _ensure_text_relations_indexes(coll: Collection) -> None: coll.create_index([("updated_at", DESCENDING)], name="updated_at_-1") +def _ensure_scoped_knowledge_assertions_indexes(coll: Collection) -> None: + """Ensure actor-scoped assertions stay bounded and cheaply retrievable.""" + + coll.create_index( + [("assertion_id", ASCENDING)], + name="assertion_id_unique", + unique=True, + ) + coll.create_index( + [ + ("subject_concept_id", ASCENDING), + ("scope.audience_keys", ASCENDING), + ("updated_at", DESCENDING), + ], + name="subject_audience_updated_at_desc", + ) + coll.create_index( + [ + ("object_concept_id", ASCENDING), + ("scope.audience_keys", ASCENDING), + ("updated_at", DESCENDING), + ], + name="object_audience_updated_at_desc", + ) + coll.create_index( + [ + ("predicate", ASCENDING), + ("scope.audience_keys", ASCENDING), + ("updated_at", DESCENDING), + ], + name="predicate_audience_updated_at_desc", + ) + + def _ensure_meta_relations_indexes(coll: Collection) -> None: existing_indexes = {idx["name"] for idx in coll.list_indexes()} if "type_1_subject_type_id_1" not in existing_indexes: @@ -1205,12 +1240,8 @@ def _ensure_meta_relations_indexes(coll: Collection) -> None: def _ensure_relationship_extent_index_indexes(coll: Collection) -> None: - existing = [ - idx for idx in coll.list_indexes() if isinstance(idx, Mapping) - ] - existing_names = { - str(idx.get("name") or "") for idx in existing if idx.get("name") - } + existing = [idx for idx in coll.list_indexes() if isinstance(idx, Mapping)] + existing_names = {str(idx.get("name") or "") for idx in existing if idx.get("name")} if "relation_id_1_unique" not in existing_names: coll.create_index( [("relation_id", ASCENDING)], @@ -1474,6 +1505,19 @@ def get_text_relations_collection() -> Collection | None: return None +def get_scoped_knowledge_assertions_collection() -> Collection | None: + """Return the non-canonical, actor-scoped knowledge assertion collection.""" + + db = get_db() + if db is not None: + return _ensure_collection_indexes_once( + db, + SCOPED_KNOWLEDGE_ASSERTIONS_COLLECTION_NAME, + _ensure_scoped_knowledge_assertions_indexes, + ) + return None + + def get_meta_relations_collection() -> Collection | None: """Returns the 'meta_relations' collection instance and ensures indexes. Stores auxiliary mapping documents for relation elicitation: diff --git a/src/backend/integrations/internal_mcp/catalogue.py b/src/backend/integrations/internal_mcp/catalogue.py index cee33b962..cbea1c02b 100644 --- a/src/backend/integrations/internal_mcp/catalogue.py +++ b/src/backend/integrations/internal_mcp/catalogue.py @@ -126,6 +126,17 @@ def _get_concept_by_concept_id(**kwargs): concept = enrich_concept_with_text_relations(concept) + from ...services.scoped_assertion_service import ( + list_visible_scoped_assertions, + ) + + scoped_assertions = list_visible_scoped_assertions( + subject_concept_ids=[concept_id], + limit=100, + ) + concept["scoped_assertions"] = scoped_assertions + concept["scoped_assertion_count"] = len(scoped_assertions) + # Detect vacuous typing (soft warning for agents to repair) vacuous_warning = detect_vacuous_typing(concept) if vacuous_warning: @@ -1118,8 +1129,7 @@ def _create_concepts(**kwargs): for concept_data in concepts if not ( isinstance(concept_data, dict) - and str(concept_data.get("kind") or "type").strip().lower() - == "predicate" + and str(concept_data.get("kind") or "type").strip().lower() == "predicate" ) } if ( @@ -1182,8 +1192,7 @@ def _identity_persistence_outcome( effect_status = "succeeded" if "changed" in receipt and ( - isinstance(receipt.get("changed"), bool) - or receipt.get("changed") is None + isinstance(receipt.get("changed"), bool) or receipt.get("changed") is None ): changed = receipt.get("changed") else: @@ -1192,10 +1201,7 @@ def _identity_persistence_outcome( isinstance(writes, list) and any( isinstance(write, Mapping) - and ( - write.get("relation_created") - or write.get("context_updated") - ) + and (write.get("relation_created") or write.get("context_updated")) for write in writes ) ) @@ -1383,12 +1389,8 @@ def _cancelled_after_completed_items() -> bool: continue if ( - ( - identity_candidates_supplied - or identity_rejected_candidates_supplied - ) - and not external_identifiers - ): + identity_candidates_supplied or identity_rejected_candidates_supplied + ) and not external_identifiers: results.append( { "success": False, @@ -1501,9 +1503,7 @@ def _cancelled_after_completed_items() -> bool: guard_scope=duplicate_match.guard_scope, match_source=duplicate_match.match_source, ) - if duplicate_match.match_source.startswith( - "external_identifier:" - ): + if duplicate_match.match_source.startswith("external_identifier:"): # Duplicate resolution is read-only and may consume the # remaining transport budget. Do not begin the marker # repair after cancellation. @@ -1518,9 +1518,7 @@ def _cancelled_after_completed_items() -> bool: identity_changed, identity_failures, identity_indeterminate_failures, - ) = _identity_persistence_outcome( - identity_persistence - ) + ) = _identity_persistence_outcome(identity_persistence) result["effect_status"] = identity_effect_status result["changed"] = identity_changed result["external_identity"] = { @@ -1561,19 +1559,16 @@ def _cancelled_after_completed_items() -> bool: ), } ) - elif ( - identity_failures - or identity_effect_status in {"failed", "partial"} - ): - persistence_failures = ( - identity_failures - or [ - { - "stage": "external_identity_persistence", - "outcome": identity_effect_status, - } - ] - ) + elif identity_failures or identity_effect_status in { + "failed", + "partial", + }: + persistence_failures = identity_failures or [ + { + "stage": "external_identity_persistence", + "outcome": identity_effect_status, + } + ] result.update( { "success": False, @@ -1740,19 +1735,16 @@ def _cancelled_after_completed_items() -> bool: } ] ) - elif ( - identity_failures - or identity_effect_status in {"failed", "partial"} - ): - persistence_failures = ( - identity_failures - or [ - { - "stage": "external_identity_persistence", - "outcome": identity_effect_status, - } - ] - ) + elif identity_failures or identity_effect_status in { + "failed", + "partial", + }: + persistence_failures = identity_failures or [ + { + "stage": "external_identity_persistence", + "outcome": identity_effect_status, + } + ] result["partial_failures"] = persistence_failures if isinstance(result.get("concept"), dict): concept_partial_failures = result["concept"].setdefault( @@ -1760,9 +1752,7 @@ def _cancelled_after_completed_items() -> bool: [], ) if isinstance(concept_partial_failures, list): - concept_partial_failures.extend( - persistence_failures - ) + concept_partial_failures.extend(persistence_failures) result["effect_status"] = "partial" result["changed"] = True results.append(result) @@ -1912,9 +1902,7 @@ def _collect_item_failures(field: str) -> list[dict[str, Any]]: { "success": False, "effect_status": ( - "indeterminate" - if effect_status == "indeterminate" - else "partial" + "indeterminate" if effect_status == "indeterminate" else "partial" ), "error_code": "handler_cancelled_after_partial_completion", "error": ( @@ -2079,6 +2067,86 @@ def _upsert_text_relation(**kwargs): ) +def _upsert_scoped_assertion(**kwargs): + from ...services.rag_text_relation_change_hook_service import ( + maybe_sync_concept_text_relations_to_rag, + ) + from ...services.scoped_assertion_service import upsert_scoped_assertion + + subject_concept_id = kwargs.get("subject_concept_id") + predicate = kwargs.get("predicate") + if not subject_concept_id: + return make_error_response( + "missing_parameter", + "Missing 'subject_concept_id' parameter", + details={"missing": ["subject_concept_id"]}, + ) + if not predicate: + return make_error_response( + "missing_parameter", + "Missing 'predicate' parameter", + details={"missing": ["predicate"]}, + ) + try: + result = upsert_scoped_assertion(**kwargs) + assertion = result.get("assertion") + if isinstance(assertion, dict) and assertion.get("object_kind") == "text": + maybe_sync_concept_text_relations_to_rag( + namespace=kwargs.get("namespace"), + concept_id=str(subject_concept_id), + predicate=str(assertion.get("predicate") or predicate), + ) + return result + except PermissionError as exc: + return make_error_response( + "access_denied", + str(exc), + details={"subject_concept_id": subject_concept_id}, + ) + except ValueError as exc: + return make_error_response( + "invalid_scoped_assertion", + str(exc), + details={"subject_concept_id": subject_concept_id}, + ) + except Exception as exc: + return _indeterminate_effect_error( + "The scoped-assertion operation raised unexpectedly.", + details={ + "exception_type": type(exc).__name__, + "exception": str(exc), + "subject_concept_id": subject_concept_id, + }, + ) + + +def _list_scoped_assertions(**kwargs): + from ...services.scoped_assertion_service import ( + list_visible_scoped_assertions, + ) + + subject_concept_id = kwargs.get("subject_concept_id") + predicates = kwargs.get("predicates") + predicate = kwargs.get("predicate") + if predicate and not predicates: + predicates = [predicate] + assertions = list_visible_scoped_assertions( + subject_concept_ids=([str(subject_concept_id)] if subject_concept_id else None), + argument_concept_id=kwargs.get("argument_concept_id"), + predicates=predicates, + object_kind=kwargs.get("object_kind"), + limit=kwargs.get("limit") or 200, + user_concept_id=kwargs.get("acting_user_concept_id"), + organisation_concept_id=kwargs.get("organisation_concept_id"), + ) + return { + "success": True, + "assertions": assertions, + "assertions_found": len(assertions), + "canonical_publication": False, + } + + def _get_text_relations(**kwargs): from ...services.text_value_service import get_texts_for_concept @@ -2644,19 +2712,13 @@ def _add_relationship(**kwargs): reference_concept_id = predicate_ref.get("concept_id") reference_name = predicate_ref.get("name") has_concept_id = bool( - isinstance(reference_concept_id, str) - and reference_concept_id.strip() - ) - has_name = bool( - isinstance(reference_name, str) and reference_name.strip() + isinstance(reference_concept_id, str) and reference_concept_id.strip() ) + has_name = bool(isinstance(reference_name, str) and reference_name.strip()) if has_concept_id == has_name: return make_error_response( "invalid_predicate_reference", - ( - "predicate_ref must provide exactly one of concept_id or " - "name." - ), + ("predicate_ref must provide exactly one of concept_id or " "name."), details={ "required_choice": ["concept_id", "name"], }, @@ -2680,9 +2742,9 @@ def _add_relationship(**kwargs): ), details={"on_missing": on_missing}, ) - predicate_value_kind = str( - predicate_ref.get("value_kind") or "concept" - ).strip().lower() + predicate_value_kind = ( + str(predicate_ref.get("value_kind") or "concept").strip().lower() + ) if predicate_value_kind not in {"concept", "text"}: return make_error_response( "invalid_predicate_reference", @@ -2826,9 +2888,11 @@ def _cancellation_after_predicate_dependency() -> dict[str, Any] | None: predicate_dependency_description = ( raw_dependency_description.strip() or None ) - predicate_value_kind = str( - predicate_if_missing.get("value_kind") or predicate_value_kind - ).strip().lower() + predicate_value_kind = ( + str(predicate_if_missing.get("value_kind") or predicate_value_kind) + .strip() + .lower() + ) if predicate_value_kind not in {"concept", "text"}: return make_error_response( "invalid_predicate_if_missing", @@ -2890,17 +2954,13 @@ def _cancellation_after_predicate_dependency() -> dict[str, Any] | None: resolution_status = str( resolution_payload.get("status") or "not_found" ).strip() - resolved_predicate_id = resolution_payload.get( - "resolved_concept_id" - ) + resolved_predicate_id = resolution_payload.get("resolved_concept_id") predicate_resolution = { "status": resolution_status, "requested": predicate_str, "resolved_concept_id": resolved_predicate_id, "match": resolution_payload.get("match"), - "candidates": list( - resolution_payload.get("candidates") or [] - )[:5], + "candidates": list(resolution_payload.get("candidates") or [])[:5], } if ( resolution_status == "resolved" @@ -2910,9 +2970,7 @@ def _cancellation_after_predicate_dependency() -> dict[str, Any] | None: predicate_str = resolved_predicate_id predicate_normalised = predicate_str[3:] elif resolution_status == "ambiguous": - candidates = list( - resolution_payload.get("candidates") or [] - )[:5] + candidates = list(resolution_payload.get("candidates") or [])[:5] response = make_error_response( "predicate_reference_ambiguous", ( @@ -3357,10 +3415,7 @@ def _cancellation_after_predicate_dependency() -> dict[str, Any] | None: response["predicate_dependency"] = predicate_dependency if predicate_resolution is not None: response["predicate_resolution"] = predicate_resolution - if ( - predicate_dependency_changed - or predicate_dependency_partial_failures - ): + if predicate_dependency_changed or predicate_dependency_partial_failures: response.update( { "effect_status": "partial", @@ -3479,9 +3534,7 @@ def _cancellation_after_predicate_dependency() -> dict[str, Any] | None: } ] if predicate_dependency_partial_failures: - response["partial_failures"] = list( - predicate_dependency_partial_failures - ) + response["partial_failures"] = list(predicate_dependency_partial_failures) return response @@ -8703,6 +8756,84 @@ def _upsert_text_relation_output_schema() -> Schema: ) +def _upsert_scoped_assertion_input_schema() -> Schema: + return Schema( + required={ + "subject_concept_id": str, + "predicate": str, + }, + optional={ + "target_text": (str, type(None)), + "target_concept_id": (str, type(None)), + "language": (str, type(None)), + "scope_mode": (str, type(None)), + "evidence": (dict, type(None)), + }, + allow_unknown=True, + description=( + "upsert_scoped_assertion input: subject_concept_id and predicate, " + "exactly one of target_text or target_concept_id, optional language, " + "scope_mode ('user' default or 'organisation'), and evidence. Actor, " + "organisation, namespace, turn, and non-publication are server-bound." + ), + ) + + +def _upsert_scoped_assertion_output_schema() -> Schema: + return Schema( + required={ + "success": bool, + }, + optional={ + "effect_status": (str, type(None)), + "changed": (bool, type(None)), + "assertion_id": (str, type(None)), + "assertion": (dict, type(None)), + "canonical_read_back": (dict, type(None)), + "canonical_publication": (bool, type(None)), + "storage_surface": (str, type(None)), + "error": (str, type(None)), + }, + allow_unknown=True, + description=( + "Scoped assertion write receipt with durable read-back. " + "canonical_publication is always false." + ), + ) + + +def _list_scoped_assertions_input_schema() -> Schema: + return Schema( + required={}, + optional={ + "subject_concept_id": (str, type(None)), + "argument_concept_id": (str, type(None)), + "predicate": (str, type(None)), + "predicates": (list, type(None)), + "object_kind": (str, type(None)), + "limit": (int, type(None)), + }, + allow_unknown=True, + description=( + "List assertions visible in the trusted user/organisation scope, " + "optionally filtered by subject, argument, predicate, or object kind." + ), + ) + + +def _list_scoped_assertions_output_schema() -> Schema: + return Schema( + required={ + "success": bool, + "assertions": list, + "assertions_found": int, + "canonical_publication": bool, + }, + optional={}, + allow_unknown=True, + ) + + def _get_text_relations_input_schema() -> Schema: return Schema( required={ @@ -11127,15 +11258,9 @@ def _finalise_report(report: dict[str, Any]) -> dict[str, Any]: and payload_user_candidate != trusted_user ): mismatch_fields.append("user_concept_id") - if ( - payload_org_candidate is not None - and payload_org_candidate != trusted_org - ): + if payload_org_candidate is not None and payload_org_candidate != trusted_org: mismatch_fields.append("organisation_concept_id") - if ( - explicit_namespace is not None - and explicit_namespace != trusted_namespace - ): + if explicit_namespace is not None and explicit_namespace != trusted_namespace: mismatch_fields.append("namespace") user_candidate = trusted_user @@ -11651,9 +11776,7 @@ def _summarise_effect_observation_journal( } entry[phase_name] = phase_projection entry["available_phases"] = available_phases - entry["latest_phase"] = ( - available_phases[-1] if available_phases else None - ) + entry["latest_phase"] = available_phases[-1] if available_phases else None late_terminal = raw_entry.get("late_terminal") turn_terminal = raw_entry.get("turn_terminal") outcome_resolved = False @@ -11671,8 +11794,7 @@ def _summarise_effect_observation_journal( receipt = turn_terminal.get("receipt") outcome_resolved = bool( isinstance(transport, Mapping) - and transport.get("outcome") - not in {None, "timed_out"} + and transport.get("outcome") not in {None, "timed_out"} and turn_terminal.get("effect_status") not in {None, "indeterminate", "unknown"} and isinstance(receipt, Mapping) @@ -15398,8 +15520,16 @@ def _bounded_delegated_telemetry_payload( max_limit = 75_000 raw_offset = arguments.get("offset", 0) raw_limit = arguments.get("limit", default_limit) - offset = raw_offset if isinstance(raw_offset, int) and not isinstance(raw_offset, bool) else 0 - limit = raw_limit if isinstance(raw_limit, int) and not isinstance(raw_limit, bool) else default_limit + offset = ( + raw_offset + if isinstance(raw_offset, int) and not isinstance(raw_offset, bool) + else 0 + ) + limit = ( + raw_limit + if isinstance(raw_limit, int) and not isinstance(raw_limit, bool) + else default_limit + ) offset = max(0, offset) limit = min(max_limit, max(1, limit)) canonical_json = json.dumps( @@ -15424,11 +15554,7 @@ def _bounded_delegated_telemetry_payload( "next_offset": end if end < total_chars else None, "json_chunk": canonical_json[offset:end] if offset <= total_chars else "", } - if ( - preserve_inline_below_limit - and offset == 0 - and total_chars <= limit - ): + if preserve_inline_below_limit and offset == 0 and total_chars <= limit: return { **dict(payload), "bounded_read": { @@ -15751,9 +15877,7 @@ def _turn_execution_get_live_progress(**kwargs): anonymous_session_id=( None if authorisation.get("delegated") - else _clean_optional_string( - effective_kwargs.get("anonymous_session_id") - ) + else _clean_optional_string(effective_kwargs.get("anonymous_session_id")) ), scope_key=( None @@ -15778,12 +15902,8 @@ def _turn_execution_get_live_progress(**kwargs): delegated_actor = _normalise_optional_concept_id( effective_kwargs.get("user_concept_id") ) - expected_scope_key = ( - f"user:{delegated_actor}" if delegated_actor else None - ) - resolved_scope_key = _clean_optional_string( - payload.get("resolved_scope_key") - ) + expected_scope_key = f"user:{delegated_actor}" if delegated_actor else None + resolved_scope_key = _clean_optional_string(payload.get("resolved_scope_key")) if expected_scope_key and resolved_scope_key != expected_scope_key: return make_error_response( "READ_DELEGATION_CANONICAL_TARGET_MISMATCH", @@ -15796,9 +15916,7 @@ def _turn_execution_get_live_progress(**kwargs): "bound": expected_scope_key, } ], - "identifier_binding": authorisation.get( - "identifier_binding" - ), + "identifier_binding": authorisation.get("identifier_binding"), }, ) from ...services.conversation_scope_binding_service import ( @@ -15821,9 +15939,7 @@ def _turn_execution_get_live_progress(**kwargs): effective_kwargs.get("session_id") ), history_owner_user_id=_normalise_optional_concept_id( - effective_kwargs.get( - "_delegated_history_owner_user_id" - ) + effective_kwargs.get("_delegated_history_owner_user_id") ), read_namespace=_clean_optional_string( effective_kwargs.get("_delegated_read_namespace") @@ -15979,9 +16095,7 @@ def _turn_execution_get_diagnostics(**kwargs): ), ( "history_owner_user_id", - _normalise_optional_concept_id( - payload.get("derived_user_concept_id") - ), + _normalise_optional_concept_id(payload.get("derived_user_concept_id")), _normalise_optional_concept_id( effective_kwargs.get("_delegated_history_owner_user_id") ), @@ -16014,9 +16128,7 @@ def _turn_execution_get_diagnostics(**kwargs): "Canonical turn diagnostics do not match the signed delegation target", details={ "field_mismatches": canonical_mismatches, - "identifier_binding": authorisation.get( - "identifier_binding" - ), + "identifier_binding": authorisation.get("identifier_binding"), }, ) payload["read_delegation"] = authorisation.get("read_delegation") @@ -16271,9 +16383,7 @@ def _context_bundle_build_benchmark(**kwargs): def _testing_theory_create_slice(**kwargs): - if denial := _internal_mcp_operator_control_plane_denial( - "testing control plane" - ): + if denial := _internal_mcp_operator_control_plane_denial("testing control plane"): return denial from ...services.testing_theory_service import create_testing_theory_slice @@ -18800,8 +18910,7 @@ def _internal_mcp_actor_scoped_read_denial( if source == "trusted_operator_payload_fallback": return None if source is None and not bool( - get_effective_user_concept_id() - or get_effective_organisation_concept_id() + get_effective_user_concept_id() or get_effective_organisation_concept_id() ): # Preserve explicit in-process operator/startup calls. Gateway and # proxy callers always bind a source and cannot reach this branch. @@ -18891,11 +19000,11 @@ def _authorise_internal_mcp_telemetry_read( if not verified.get("success"): return make_error_response( str(verified.get("error_code") or "INVALID_CONTEXT_BINDING"), - str(verified.get("error_message") or "Telemetry read delegation is invalid"), + str( + verified.get("error_message") or "Telemetry read delegation is invalid" + ), details={ - "identifier_binding": dict( - verified.get("identifier_binding") or {} - ) + "identifier_binding": dict(verified.get("identifier_binding") or {}) }, ) @@ -18903,9 +19012,7 @@ def _authorise_internal_mcp_telemetry_read( actor_user_id = _normalise_optional_concept_id( verified.get("delegated_actor_user_id") ) - actor_namespace = _clean_optional_string( - verified.get("delegated_actor_namespace") - ) + actor_namespace = _clean_optional_string(verified.get("delegated_actor_namespace")) organisation_concept_id = _normalise_optional_concept_id( verified.get("organisation_concept_id") ) @@ -18922,16 +19029,13 @@ def _authorise_internal_mcp_telemetry_read( ), ( "organisation_concept_id", - _normalise_optional_concept_id( - payload.get("organisation_concept_id") - ), + _normalise_optional_concept_id(payload.get("organisation_concept_id")), organisation_concept_id, ), ( "session_id", _clean_optional_string( - payload.get("session_id") - or payload.get("conversation_session_id") + payload.get("session_id") or payload.get("conversation_session_id") ), bound_session_id, ), @@ -18947,11 +19051,7 @@ def _authorise_internal_mcp_telemetry_read( if isinstance(payload.get("history_index"), int) else None ), - ( - bound_history_index - if isinstance(bound_history_index, int) - else None - ), + (bound_history_index if isinstance(bound_history_index, int) else None), ), ) mismatches = [ @@ -18988,9 +19088,7 @@ def _authorise_internal_mcp_telemetry_read( effective["request_id"] = bound_request_id if isinstance(bound_history_index, int): effective["history_index"] = bound_history_index - effective["_verified_read_delegation"] = dict( - verified.get("read_delegation") or {} - ) + effective["_verified_read_delegation"] = dict(verified.get("read_delegation") or {}) effective["_verified_identifier_binding"] = dict( verified.get("identifier_binding") or {} ) @@ -23791,8 +23889,7 @@ def _rag_get_item(**kwargs): direct_tool_invocations = doc.get("tool_invocations") tool_invocations = ( direct_tool_invocations - if isinstance(direct_tool_invocations, list) - and direct_tool_invocations + if isinstance(direct_tool_invocations, list) and direct_tool_invocations else execution_payload.get("tool_invocations") ) discovery_payload_raw = workflow_routing_diagnostics.get("discovery") @@ -23833,15 +23930,11 @@ def _rag_get_item(**kwargs): ( late_effect_observations, late_effect_observation_count, - ) = _summarise_late_effect_observations( - doc.get("late_effect_observations") - ) + ) = _summarise_late_effect_observations(doc.get("late_effect_observations")) ( effect_observation_journal, effect_observation_journal_count, - ) = _summarise_effect_observation_journal( - doc.get("effect_observation_journal") - ) + ) = _summarise_effect_observation_journal(doc.get("effect_observation_journal")) payload = { "collection": collection, @@ -23889,13 +23982,10 @@ def _rag_get_item(**kwargs): "late_effect_observations_truncated": ( late_effect_observation_count > len(late_effect_observations) ), - "effect_observation_journal_count": ( - effect_observation_journal_count - ), + "effect_observation_journal_count": (effect_observation_journal_count), "effect_observation_journal": effect_observation_journal, "effect_observation_journal_truncated": ( - effect_observation_journal_count - > len(effect_observation_journal) + effect_observation_journal_count > len(effect_observation_journal) ), "required_effects": required_effects, "postcondition_checks": postcondition_checks, @@ -25514,9 +25604,7 @@ def _github_get_auth_config(**kwargs): "env_keys_used": { "token": token_key, "command": ( - "VON_GITHUB_MCP_COMMAND" - if env.get("VON_GITHUB_MCP_COMMAND") - else None + "VON_GITHUB_MCP_COMMAND" if env.get("VON_GITHUB_MCP_COMMAND") else None ), "args": "VON_GITHUB_MCP_ARGS" if env.get("VON_GITHUB_MCP_ARGS") else None, "allow_list": "VON_GITHUB_REPO_ALLOW_LIST", @@ -28453,9 +28541,7 @@ def _sanitise_for_introspection(value: Any, *, depth: int = 0) -> Any: gateway = current_app.config.get("INTERNAL_MCP_GATEWAY") gateway_enabled = getattr(gateway, "enabled", None) - startup_status = current_app.config.get( - "INTERNAL_MCP_ORCHESTRATOR_STATUS" - ) + startup_status = current_app.config.get("INTERNAL_MCP_ORCHESTRATOR_STATUS") if isinstance(startup_status, dict): legacy_orchestrator_status = str( startup_status.get("state") or legacy_orchestrator_status @@ -28549,9 +28635,7 @@ def _sanitise_for_introspection(value: Any, *, depth: int = 0) -> Any: "automatic workflow selector or general controller on this path. Explicit " "workflows remain separately callable through their registered interfaces." ) - tool_guidance_hash = hashlib.sha256( - tool_guidance_text.encode("utf-8") - ).hexdigest() + tool_guidance_hash = hashlib.sha256(tool_guidance_text.encode("utf-8")).hexdigest() tool_guidance_preview = None if include_tool_guidance_preview and max_preview_chars_int: tool_guidance_preview = tool_guidance_text[:max_preview_chars_int] @@ -33083,6 +33167,47 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: "Supports language preferences, instance_of restriction, and optional code-string matching." ), ), + MethodDefinition( + name="upsert_scoped_assertion", + handler=_upsert_scoped_assertion, + input_schema=_upsert_scoped_assertion_input_schema(), + output_schema=_upsert_scoped_assertion_output_schema(), + category="write", + ordinary_turn_trusted_argument_bindings={ + "acting_user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + "namespace": "turn_namespace", + "turn_id": "turn_id", + }, + ordinary_turn_fixed_arguments={"canonical_publication": False}, + ordinary_turn_effect=True, + effect_admission_window_sec=8.0, + description=( + "Assert provenance-bearing user- or organisation-scoped knowledge " + "about any concept visible to the current actor, including globally " + "visible concepts. This does not alter or publish the concept's " + "canonical Vontology record. Use this instead of upsert_text_relation " + "or add_relationship when the actor can see but does not own the " + "canonical subject. Provide exactly one of target_text or " + "target_concept_id; choose scope_mode='organisation' only for " + "knowledge intended to be shared with the authenticated organisation." + ), + ), + MethodDefinition( + name="list_scoped_assertions", + handler=_list_scoped_assertions, + input_schema=_list_scoped_assertions_input_schema(), + output_schema=_list_scoped_assertions_output_schema(), + category="read", + ordinary_turn_trusted_argument_bindings={ + "acting_user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + }, + description=( + "Read provenance-bearing non-canonical assertions visible to the " + "trusted user or organisation, with bounded filters." + ), + ), MethodDefinition( name="upsert_text_relation", handler=_upsert_text_relation, @@ -33096,7 +33221,13 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: ordinary_turn_effect=True, effect_admission_window_sec=8.0, ordinary_turn_mutation_subject_argument="concept_id", - description="Add or update ANY text relation (hasContent, hasDescription, hasNote, custom predicates, etc.). Use for attaching text content to concepts with flexible predicate types. More general than add_names which is specialized for hasName relations only.", + description=( + "Add or update a canonical text relation when the actor has mutation " + "authority over the subject concept. For knowledge about a visible " + "concept the actor does not own, use upsert_scoped_assertion instead. " + "Supports hasContent, hasDescription, hasNote, hasName, and existing " + "custom predicate concepts." + ), ), MethodDefinition( name="get_text_relations", @@ -34146,9 +34277,9 @@ def _build_default_catalogue_knowledge_io_definitions() -> List[MethodDefinition return definitions -def _build_default_catalogue_external_integration_definitions() -> List[ - MethodDefinition -]: +def _build_default_catalogue_external_integration_definitions() -> ( + List[MethodDefinition] +): jira_search_output_schema = _jira_generic_output_schema("search") jira_get_issue_output_schema = _jira_generic_output_schema("get_issue") jira_get_project_issue_types_output_schema = ( @@ -34670,9 +34801,9 @@ def _build_default_catalogue_external_integration_definitions() -> List[ return definitions -def _build_default_catalogue_diagnostics_and_research_definitions() -> List[ - MethodDefinition -]: +def _build_default_catalogue_diagnostics_and_research_definitions() -> ( + List[MethodDefinition] +): definitions: List[MethodDefinition] = [ MethodDefinition( name="rag_get_status", diff --git a/src/backend/services/adaptive_turn_service.py b/src/backend/services/adaptive_turn_service.py index 8109ff109..f0a95edef 100644 --- a/src/backend/services/adaptive_turn_service.py +++ b/src/backend/services/adaptive_turn_service.py @@ -67,6 +67,7 @@ _MODEL_EVIDENCE_PREVIEW_MAX_CHARS = 240 _CAPABILITY_CATALOGUE_SCHEMA_VERSION = "adaptive_turn_capabilities.v1" + @dataclass(frozen=True) class AdaptiveTurnResult: """Route-compatible result from one ordinary adaptive turn.""" @@ -263,9 +264,7 @@ def _compact_evidence_index( removed = compacted.pop() total_bytes -= len(_json_bytes(removed)) + (1 if compacted else 0) omission["next_offset"] = base_offset + len(compacted) - omission["omitted_count"] = complete_count - int( - omission["next_offset"] - ) + omission["omitted_count"] = complete_count - int(omission["next_offset"]) omission_size = len(_json_bytes(omission)) + (1 if compacted else 0) if total_bytes + omission_size <= max_bytes: compacted.append(omission) @@ -299,11 +298,7 @@ def _capability_schema_reference( "minimum_effect_window_seconds", ) ) - compact = { - key: capability.get(key) - for key in projected_keys - if key in capability - } + compact = {key: capability.get(key) for key in projected_keys if key in capability} name = str(capability.get("name") or "").strip() if not include_metadata: compact["capability_metadata_omitted_for_model_context"] = True @@ -401,9 +396,7 @@ def assemble( ) -> dict[str, Any]: omitted_page_entry_count = max(0, len(capabilities) - len(entries)) next_offset = ( - offset + len(entries) - if omitted_page_entry_count - else raw_next_offset + offset + len(entries) if omitted_page_entry_count else raw_next_offset ) projection: dict[str, Any] = { "schema_version": "adaptive_turn_capability_page_projection.v1", @@ -507,9 +500,7 @@ def assemble( name_only = { "name": str(capability.get("name") or "").strip(), "capability_metadata_omitted_for_model_context": True, - "input_schema_omitted_for_model_context": ( - "input_schema" in capability - ), + "input_schema_omitted_for_model_context": ("input_schema" in capability), } name_only_candidate = assemble( [*included, name_only], @@ -600,9 +591,7 @@ def _bound_tool_results_for_model( if not results: return [] - receipt_outputs = [ - _model_tool_output_receipt(result.output) for result in results - ] + receipt_outputs = [_model_tool_output_receipt(result.output) for result in results] serialisable_shells = [ { "call_id": result.call_id, @@ -713,9 +702,7 @@ def _evidence_context_message( ) evidence_view_projection: dict[str, Any] = { "schema_version": "adaptive_turn_evidence_view_projection.v1", - "order": ( - "content_bearing_evidence_slices_first_stable_within_class" - ), + "order": ("content_bearing_evidence_slices_first_stable_within_class"), "deduplication": "exact_canonical_output", "total_count": total_view_count, "included_count": len(included_views), @@ -760,15 +747,10 @@ def _ordered_unique_evidence_views( continue evidence_id = item.get("evidence_id") is_index_page = ( - item.get("schema_version") - == "adaptive_turn_evidence_index_page.v1" + item.get("schema_version") == "adaptive_turn_evidence_index_page.v1" ) - if ( - not is_index_page - and ( - not isinstance(evidence_id, str) - or not evidence_id.strip() - ) + if not is_index_page and ( + not isinstance(evidence_id, str) or not evidence_id.strip() ): continue view = dict(item) @@ -909,15 +891,9 @@ def smallest_pageable_index() -> list[dict[str, Any]] | None: # it must never displace a hydrated slice that already fits. expanded_projection = largest_index_projection( included_views, - preview_max_chars=( - 0 if included_views else _MODEL_EVIDENCE_PREVIEW_MAX_CHARS - ), - ) - return ( - expanded_projection[1] - if expanded_projection is not None - else best_message + preview_max_chars=(0 if included_views else _MODEL_EVIDENCE_PREVIEW_MAX_CHARS), ) + return expanded_projection[1] if expanded_projection is not None else best_message def _final_synthesis_context( @@ -1215,8 +1191,7 @@ def _registered_capability_names( if definition is None: continue if definition.category == "read" or ( - definition.category == "write" - and definition.ordinary_turn_effect + definition.category == "write" and definition.ordinary_turn_effect ): allowed.append(name) return tuple(allowed) @@ -1242,9 +1217,7 @@ def ordinary_turn_capability_delegation( if gateway is None or not gateway.enabled: return () - authenticated = bool( - isinstance(user_concept_id, str) and user_concept_id.strip() - ) + authenticated = bool(isinstance(user_concept_id, str) and user_concept_id.strip()) trusted_values = { str(key): value for key, value in (trusted_argument_values or {}).items() @@ -1299,13 +1272,16 @@ def _capability_catalogue( ) -> dict[str, Any]: query = str(payload.get("query") or "").strip().lower() requested_names = payload.get("names") - exact_names = { - str(item).strip().lower() - for item in requested_names - if isinstance(item, str) and item.strip() - } if isinstance(requested_names, Sequence) and not isinstance( - requested_names, (str, bytes, bytearray) - ) else set() + exact_names = ( + { + str(item).strip().lower() + for item in requested_names + if isinstance(item, str) and item.strip() + } + if isinstance(requested_names, Sequence) + and not isinstance(requested_names, (str, bytes, bytearray)) + else set() + ) try: offset = max(0, int(payload.get("offset", 0))) except (TypeError, ValueError): @@ -1316,9 +1292,7 @@ def _capability_catalogue( limit = 20 query_tokens = { - token - for token in re.findall(r"[a-z0-9_]+", query) - if len(token) > 1 + token for token in re.findall(r"[a-z0-9_]+", query) if len(token) > 1 } ranked: list[tuple[int, str, dict[str, Any]]] = [] matched_total = 0 @@ -1416,9 +1390,7 @@ def _capability_catalogue( "evidence_surface_family": ( surface_metadata.evidence_surface_family ), - "external_surface": bool( - surface_metadata.external_surface - ), + "external_surface": bool(surface_metadata.external_surface), } ) ranked.append( @@ -1456,12 +1428,9 @@ def _capability_catalogue( f"{description.lower()}" ) if query_tokens: - literal_matches = sum( - 1 for token in query_tokens if token in searchable - ) + literal_matches = sum(1 for token in query_tokens if token in searchable) query_match = bool( - literal_matches - or float(capability.get("relevance_score") or 0.0) > 0.0 + literal_matches or float(capability.get("relevance_score") or 0.0) > 0.0 ) semantic_score = int( max( @@ -1499,9 +1468,7 @@ def _capability_catalogue( "success": True, "delegation": "bounded_capabilities", "total": len(selected), - "delegated_total": ( - registered_delegated_total + represented_workflow_total - ), + "delegated_total": (registered_delegated_total + represented_workflow_total), "registered_tool_total": registered_delegated_total, "represented_workflow_total": represented_workflow_total, "matched_total": matched_total, @@ -1550,16 +1517,10 @@ def _model_visible_input_schema(definition: Any) -> dict[str, Any]: schema = dict(schema_to_json_schema(definition.input_schema)) raw_properties = schema.get("properties") - properties = ( - dict(raw_properties) if isinstance(raw_properties, Mapping) else {} - ) + properties = dict(raw_properties) if isinstance(raw_properties, Mapping) else {} raw_required = schema.get("required") required = ( - [ - str(field_name) - for field_name in raw_required - if isinstance(field_name, str) - ] + [str(field_name) for field_name in raw_required if isinstance(field_name, str)] if isinstance(raw_required, Sequence) and not isinstance(raw_required, (str, bytes, bytearray)) else [] @@ -1570,15 +1531,11 @@ def _model_visible_input_schema(definition: Any) -> dict[str, Any]: hidden_arguments.update(str(argument_name) for argument_name in bindings) fixed_arguments = definition.ordinary_turn_fixed_arguments if isinstance(fixed_arguments, Mapping): - hidden_arguments.update( - str(argument_name) for argument_name in fixed_arguments - ) + hidden_arguments.update(str(argument_name) for argument_name in fixed_arguments) for argument_name in hidden_arguments: properties.pop(str(argument_name), None) required = [ - field_name - for field_name in required - if field_name != str(argument_name) + field_name for field_name in required if field_name != str(argument_name) ] aliases = schema.get("x-von-argument-aliases") if isinstance(aliases, Mapping): @@ -1830,9 +1787,13 @@ def walk(value: Any, *, depth: int) -> None: key = str(raw_key).strip().lower() if key in scalar_fields: append_value(item) - elif key in sequence_fields and isinstance(item, Sequence) and not isinstance( - item, - (str, bytes, bytearray), + elif ( + key in sequence_fields + and isinstance(item, Sequence) + and not isinstance( + item, + (str, bytes, bytearray), + ) ): for nested in list(item)[:50]: append_value(nested) @@ -1877,7 +1838,9 @@ def _effect_status( return explicit if isinstance(raw_payload, Mapping) and raw_payload.get("success") is False: return "failed" - return "failed" if bool(getattr(transport_result, "timed_out", False)) else "succeeded" + return ( + "failed" if bool(getattr(transport_result, "timed_out", False)) else "succeeded" + ) def _late_effect_observation_state( @@ -1895,14 +1858,15 @@ def _late_effect_observation_state( effect_status = _effect_status(payload, transport_result=None) changed = ( payload.get("changed") - if isinstance(payload, Mapping) - and isinstance(payload.get("changed"), bool) + if isinstance(payload, Mapping) and isinstance(payload.get("changed"), bool) else (False if effect_status in {"failed", "not_started"} else None) ) return effect_status, changed -def _error_payload(code: str, message: str, *, retryable: bool = False) -> dict[str, Any]: +def _error_payload( + code: str, message: str, *, retryable: bool = False +) -> dict[str, Any]: return { "success": False, "error_code": code, @@ -2014,6 +1978,7 @@ def execute_adaptive_turn( "turn_namespace": scope.namespace, "actor_user_concept_id": scope.user_concept_id, "actor_organisation_concept_id": scope.organisation_concept_id, + "turn_id": turn_id or "ordinary-turn", } ) delegated_names = ordinary_turn_capability_delegation( @@ -2044,9 +2009,7 @@ def execute_adaptive_turn( "schema_version": "adaptive_turn_budget_allocation.v1", "turn_budget_seconds": turn_budget, "final_synthesis_reserve_seconds": final_reserve, - "requested_final_answer_reserve_seconds": ( - requested_final_answer_reserve - ), + "requested_final_answer_reserve_seconds": (requested_final_answer_reserve), "effective_final_answer_reserve_seconds": final_answer_reserve, "final_answer_reserve_source": ( "caller" @@ -2227,12 +2190,8 @@ def retain_model_evidence_views(results: Sequence[ToolResult]) -> None: result.output.get("schema_version") == "adaptive_turn_evidence_index_page.v1" ) - if ( - not is_index_page - and ( - not isinstance(evidence_id, str) - or not evidence_id.strip() - ) + if not is_index_page and ( + not isinstance(evidence_id, str) or not evidence_id.strip() ): continue view = dict(result.output) @@ -2245,8 +2204,7 @@ def retain_model_evidence_views(results: Sequence[ToolResult]) -> None: def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: with effect_state_lock: effect_snapshot = { - effect_id: dict(state) - for effect_id, state in effect_states.items() + effect_id: dict(state) for effect_id, state in effect_states.items() } incomplete_effects = [ state @@ -2256,8 +2214,7 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: ] if status == "completed" and incomplete_effects: incomplete_statuses = { - str(state.get("effect_status") or "") - for state in incomplete_effects + str(state.get("effect_status") or "") for state in incomplete_effects } if "indeterminate" in incomplete_statuses: status = "effect_outcome_indeterminate" @@ -2272,9 +2229,7 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: invocation = dict(raw_invocation) effect_id = invocation.get("effect_id") state = ( - effect_snapshot.get(effect_id) - if isinstance(effect_id, str) - else None + effect_snapshot.get(effect_id) if isinstance(effect_id, str) else None ) if isinstance(state, Mapping): invocation["effect_status"] = state.get("effect_status") @@ -2284,9 +2239,7 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: if state.get("evidence_id"): invocation["late_evidence_id"] = state.get("evidence_id") if isinstance(state.get("late_observation"), Mapping): - invocation["late_completion"] = dict( - state["late_observation"] - ) + invocation["late_completion"] = dict(state["late_observation"]) reconciled_invocations.append(invocation) relevant_effects = [ @@ -2367,9 +2320,7 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: aux_calls.append( { "type": "adaptive_turn_effect_finality_fallback", - "schema_version": ( - "adaptive_turn_effect_finality_fallback.v1" - ), + "schema_version": ("adaptive_turn_effect_finality_fallback.v1"), "terminal_status": status, "effect_count": len(relevant_effects), "status_counts": status_counts, @@ -2466,21 +2417,16 @@ def enter_answer_only(reason: str) -> None: aux_calls.append( { "type": "adaptive_turn_final_answer_reserve_entered", - "schema_version": ( - "adaptive_turn_final_answer_reserve_entered.v1" - ), + "schema_version": ("adaptive_turn_final_answer_reserve_entered.v1"), "reason": reason, - "requested_answer_reserve_seconds": ( - requested_final_answer_reserve - ), + "requested_answer_reserve_seconds": (requested_final_answer_reserve), "effective_answer_reserve_seconds": final_answer_reserve, "final_synthesis_reserve_seconds": final_reserve, "evidence_capable_reserve_seconds": ( final_reserve - final_answer_reserve ), "reserve_clamped": ( - final_answer_reserve - < requested_final_answer_reserve + final_answer_reserve < requested_final_answer_reserve ), "remaining_ms": max( 0.0, @@ -2544,11 +2490,7 @@ def enter_answer_only(reason: str) -> None: stage_deadline = ( turn_deadline if answer_only - else ( - final_answer_deadline - if final_synthesis - else research_deadline - ) + else (final_answer_deadline if final_synthesis else research_deadline) ) effective_params = dict(model_parameters or {}) effective_params["request_timeout_seconds"] = max( @@ -2560,11 +2502,7 @@ def enter_answer_only(reason: str) -> None: request_tools = ( [] if answer_only - else ( - final_synthesis_tools - if final_synthesis - else available_tools - ) + else (final_synthesis_tools if final_synthesis else available_tools) ) try: response: LLMResponse = llm_client.generate_with_tools( @@ -2575,8 +2513,7 @@ def enter_answer_only(reason: str) -> None: system_message=_scope_message( scope, delegated_count=( - len(delegated_names) - + len(workflow_capabilities_by_name) + len(delegated_names) + len(workflow_capabilities_by_name) ), final_synthesis=final_synthesis, answer_only=answer_only, @@ -2785,9 +2722,7 @@ def enter_answer_only(reason: str) -> None: "stage": stage, "mode": mode, "model": response.model or model, - "request_timeout_seconds": effective_params[ - "request_timeout_seconds" - ], + "request_timeout_seconds": effective_params["request_timeout_seconds"], "duration_ms": call_duration_ms, "usage": dict(response.usage) if response.usage else None, "status": "completed", @@ -2824,9 +2759,7 @@ def enter_answer_only(reason: str) -> None: aux_calls.append( { "type": "adaptive_turn_expired_final_evidence_calls", - "schema_version": ( - "adaptive_turn_expired_final_evidence_calls.v1" - ), + "schema_version": ("adaptive_turn_expired_final_evidence_calls.v1"), "discarded_tool_call_count": len(calls), } ) @@ -2911,9 +2844,7 @@ def enter_answer_only(reason: str) -> None: "total": len(complete_index), "offset": offset, "next_offset": ( - next_offset - if next_offset < len(complete_index) - else None + next_offset if next_offset < len(complete_index) else None ), "evidence": compact_page, } @@ -3114,9 +3045,7 @@ def enter_answer_only(reason: str) -> None: else str(workflow_capability.name) ) execution_method_name = ( - canonical_name - if canonical_name is not None - else "workflow_execute" + canonical_name if canonical_name is not None else "workflow_execute" ) definition = gateway.get_method_definition(execution_method_name) is_workflow_capability = workflow_capability is not None @@ -3169,9 +3098,7 @@ def enter_answer_only(reason: str) -> None: is_effect=True, execution_method_name=execution_method_name, capability_kind="represented_workflow", - represented_workflow_id=str( - workflow_capability.workflow_id - ), + represented_workflow_id=str(workflow_capability.workflow_id), ) continue if not scope.user_concept_id or not scope.namespace: @@ -3189,9 +3116,7 @@ def enter_answer_only(reason: str) -> None: is_effect=True, execution_method_name=execution_method_name, capability_kind="represented_workflow", - represented_workflow_id=str( - workflow_capability.workflow_id - ), + represented_workflow_id=str(workflow_capability.workflow_id), ) continue from src.backend.services.workflow_turn_capability_service import ( @@ -3228,15 +3153,11 @@ def enter_answer_only(reason: str) -> None: is_effect=True, execution_method_name=execution_method_name, capability_kind="represented_workflow", - represented_workflow_id=str( - workflow_capability.workflow_id - ), + represented_workflow_id=str(workflow_capability.workflow_id), ) continue if turn_id: - stable_launch_inputs = dict( - trusted_arguments.get("inputs") or {} - ) + stable_launch_inputs = dict(trusted_arguments.get("inputs") or {}) # Tool results appended to the adaptive context must not # change the identity of an otherwise identical retry. stable_launch_inputs.pop("augmented_context", None) @@ -3260,9 +3181,7 @@ def enter_answer_only(reason: str) -> None: binding_diagnostics = { **dict(binding_diagnostics or {}), "durable_idempotency": { - "schema_version": ( - "workflow_turn_durable_idempotency.v1" - ), + "schema_version": ("workflow_turn_durable_idempotency.v1"), "source_event_type": "conversation_turn", "source_event_id": turn_id, "event_idempotency_key_source": ( @@ -3386,8 +3305,7 @@ def contained( ) if ( prior_terminal_failure is not None - and prior_terminal_failure[0] - == successful_effect_mutation_generation + and prior_terminal_failure[0] == successful_effect_mutation_generation ): return index, contained( { @@ -3405,12 +3323,8 @@ def contained( "prior_error_code": prior_terminal_failure[1], "changed": False, "recovery_affordances": [ - { - "action_type": "change_arguments_or_use_typed_recovery" - }, - { - "action_type": "inspect_canonical_state_before_retry" - }, + {"action_type": "change_arguments_or_use_typed_recovery"}, + {"action_type": "inspect_canonical_state_before_retry"}, ], } ) @@ -3450,8 +3364,7 @@ def contained( "mutation_outcome": "not_started", "outcome_finality": "terminal_for_turn", "persistence_reason": ( - dispatch_outcome.get("reason") - or "not_acknowledged" + dispatch_outcome.get("reason") or "not_acknowledged" ), "recovery_affordances": [ { @@ -3600,9 +3513,7 @@ def contained( None, ) transport_metadata = ( - transport_metadata_fn() - if callable(transport_metadata_fn) - else {} + transport_metadata_fn() if callable(transport_metadata_fn) else {} ) assert terminal_effect_status is not None changed = ( @@ -3641,16 +3552,13 @@ def contained( "effect_id": effect_identifier, "phase": "turn_terminal", "reason": ( - terminal_outcome.get("reason") - or "not_acknowledged" + terminal_outcome.get("reason") or "not_acknowledged" ), } ) return index, contained(raw_payload, transport_result) - if actual_capabilities and any( - item.is_effect for item in actual_capabilities - ): + if actual_capabilities and any(item.is_effect for item in actual_capabilities): # Preserve model-call order whenever the batch contains an effect. # Each effect receives an independent admission decision in model # order. One invalid or oversized call therefore cannot deny an @@ -3680,11 +3588,7 @@ def contained( "prior_effect_id": prior_indeterminate_effect_id, "capability_name": canonical_name, "recovery_affordances": [ - { - "action_type": ( - "inspect_canonical_state_before_retry" - ) - } + {"action_type": ("inspect_canonical_state_before_retry")} ], } result_index, contained = invoke_and_contain( @@ -3759,9 +3663,7 @@ def contained( None, ) transport_metadata = ( - transport_metadata_fn() - if callable(transport_metadata_fn) - else {} + transport_metadata_fn() if callable(transport_metadata_fn) else {} ) effect_status = ( _effect_status( @@ -3946,9 +3848,7 @@ def contained( ) correlated_results = [ - result - for result in batch_results - if isinstance(result, ToolResult) + result for result in batch_results if isinstance(result, ToolResult) ] if len(correlated_results) != len(calls): terminal_status = "tool_result_correlation_error" @@ -3962,9 +3862,7 @@ def contained( aux_calls.append( { "type": "adaptive_turn_tool_result_batch_overflow", - "schema_version": ( - "adaptive_turn_tool_result_batch_overflow.v1" - ), + "schema_version": ("adaptive_turn_tool_result_batch_overflow.v1"), "tool_call_count": len(correlated_results), "max_bytes": _MODEL_TOOL_RESULT_BATCH_MAX_BYTES, "action": ( diff --git a/src/backend/services/concept_relation_service.py b/src/backend/services/concept_relation_service.py index f83908b33..fb24662ae 100644 --- a/src/backend/services/concept_relation_service.py +++ b/src/backend/services/concept_relation_service.py @@ -348,9 +348,7 @@ def find_relations_with_argument( ) incoming_asserted_binary_diagnostics = { "requested": bool( - include_asserted_rows - and include_structural - and include_arg2_or_later + include_asserted_rows and include_structural and include_arg2_or_later ), "path": "not_requested", "used_relationship_extent_index": False, @@ -479,9 +477,7 @@ def find_relations_with_argument( ) for candidate in candidate_rows: if candidate[0] not in accessible_source_ids: - incoming_asserted_binary_diagnostics[ - "rows_filtered_by_access" - ] += 1 + incoming_asserted_binary_diagnostics["rows_filtered_by_access"] += 1 continue incoming_candidates.append(candidate) else: @@ -492,18 +488,12 @@ def find_relations_with_argument( incoming_asserted_binary_diagnostics.update( { "path": "canonical_exact_predicate_query", - "fallback_reason": ( - "relationship_extent_index_unavailable" - ), + "fallback_reason": ("relationship_extent_index_unavailable"), } ) exact_query = { "$or": [ - { - f"relationships.{predicate_id}": ( - resolved_concept_id - ) - } + {f"relationships.{predicate_id}": (resolved_concept_id)} for predicate_id in exact_predicate_ids ] } @@ -519,9 +509,7 @@ def find_relations_with_argument( exact_documents = ConceptsRepository.find( exact_query, exact_projection, - max_time_ms=( - _CANONICAL_EXACT_PREDICATE_FALLBACK_MAX_TIME_MS - ), + max_time_ms=(_CANONICAL_EXACT_PREDICATE_FALLBACK_MAX_TIME_MS), ) for document in exact_documents: source_id = document.get("concept_id") @@ -555,9 +543,7 @@ def find_relations_with_argument( incoming_asserted_binary_diagnostics.update( { "path": "canonical_aggregation", - "fallback_reason": ( - "relationship_extent_index_unavailable" - ), + "fallback_reason": ("relationship_extent_index_unavailable"), } ) incoming_pipeline = [ @@ -566,9 +552,7 @@ def find_relations_with_argument( "$project": { "concept_id": 1, "updated_at": 1, - "relationship_items": { - "$objectToArray": "$relationships" - }, + "relationship_items": {"$objectToArray": "$relationships"}, } }, {"$unwind": "$relationship_items"}, @@ -583,16 +567,12 @@ def find_relations_with_argument( {"$match": {"targets": resolved_concept_id}}, ] for row in ConceptsRepository.aggregate(incoming_pipeline): - incoming_asserted_binary_diagnostics[ - "canonical_rows_returned" - ] += 1 + incoming_asserted_binary_diagnostics["canonical_rows_returned"] += 1 source_id = row.get("concept_id") if not isinstance(source_id, str) or not source_id.strip(): continue source_id = source_id.strip() - targets = _normalise_relationship_targets( - row.get("targets") - ) + targets = _normalise_relationship_targets(row.get("targets")) if not targets: continue for target_index, target_value in enumerate(targets): @@ -615,9 +595,12 @@ def find_relations_with_argument( include_preview=include_concept_preview, preview_cache=preview_cache, ) - for source_id, predicate_id, target_index, source_updated_at in ( - incoming_candidates - ): + for ( + source_id, + predicate_id, + target_index, + source_updated_at, + ) in incoming_candidates: if not _predicate_matches_terms(predicate_id, predicate_terms): continue matched_indexes = [_ARG_INDEX_FIRST_OBJECT + target_index] @@ -662,6 +645,74 @@ def find_relations_with_argument( hit["source_concept_preview"] = source_preview hits.append(hit) + if include_asserted_rows and include_structural: + from .scoped_assertion_service import list_visible_scoped_assertions + + for assertion in list_visible_scoped_assertions( + argument_concept_id=resolved_concept_id, + predicates=predicate_display_terms or None, + object_kind="concept", + limit=_MAX_LIMIT, + ): + source_id = str(assertion.get("subject_concept_id") or "") + target_id = str(assertion.get("object_concept_id") or "") + predicate_id = assertion.get("predicate") + if not source_id or not target_id: + continue + matched_indexes: List[int] = [] + if source_id == resolved_concept_id: + matched_indexes.append(_ARG_INDEX_SUBJECT) + if target_id == resolved_concept_id: + matched_indexes.append(_ARG_INDEX_FIRST_OBJECT) + if not _argument_indexes_match(matched_indexes, argument_filter): + continue + if not _predicate_matches_terms(predicate_id, predicate_terms): + continue + source_preview = _resolve_concept_preview( + source_id, + include_concept_preview, + preview_cache, + ) + hit = { + "source_concept_id": source_id, + "predicate_concept_id": predicate_id, + "relation_kind": "binary", + "argument_indexes": matched_indexes, + "target_value": target_id, + "target_concept_preview": ( + _resolve_concept_preview( + target_id, + True, + preview_cache, + ) + if include_concept_preview + else None + ), + "relation_metadata": { + "relation_id": assertion.get("assertion_id"), + "assertion_id": assertion.get("assertion_id"), + "updated_at": assertion.get("updated_at"), + "match_type": "exact", + "assertion_scope": assertion.get("scope"), + "canonical_publication": False, + "storage_surface": "scoped_knowledge_assertions", + "provenance": assertion.get("provenance"), + }, + "access_granted": source_preview is not None + or not include_concept_preview, + "follow_up_actions": _build_follow_up_actions( + [source_id, target_id], + exclude={resolved_concept_id}, + ), + "score": 1.0, + "is_asserted": True, + "relation_state": "scoped_asserted", + "canonical_publication": False, + } + if include_concept_preview: + hit["source_concept_preview"] = source_preview + hits.append(hit) + if include_asserted_rows and include_text and include_arg1: source_preview = _resolve_concept_preview( resolved_concept_id, @@ -690,13 +741,27 @@ def find_relations_with_argument( "text_value_id": rel.get("text_value_id"), "lang": rel.get("lang"), "match_type": "exact", + "assertion_id": rel.get("assertion_id"), + "assertion_scope": rel.get("assertion_scope"), + "canonical_publication": rel.get( + "canonical_publication", + True, + ), + "storage_surface": rel.get("storage_surface"), + "provenance": rel.get("provenance"), }, "access_granted": source_preview is not None or not include_concept_preview, "follow_up_actions": [], "score": 1.0, "is_asserted": True, - "relation_state": "asserted", + "relation_state": ( + "scoped_asserted" if rel.get("assertion_id") else "asserted" + ), + "canonical_publication": rel.get( + "canonical_publication", + True, + ), } if include_concept_preview: hit["source_concept_preview"] = source_preview @@ -934,9 +999,7 @@ def get_predicate_incidence( uncertainty_statuses=uncertainty_statuses, ) if type_count_options.include: - _attach_direct_type_ids_to_fast_incidence_hits( - {resolved_concept_id: hits} - ) + _attach_direct_type_ids_to_fast_incidence_hits({resolved_concept_id: hits}) predicate_rows = _aggregate_predicate_incidence_rows( hits=hits, include_concept_preview=include_concept_preview, @@ -1033,8 +1096,10 @@ def get_predicate_incidence( uncertainty_mode=uncertainty_mode, uncertainty_statuses=uncertainty_statuses, ) - retrieval_strategy = "batched_subject_asserted" if grouped_hits is not None else ( - "per_instance_relation_lookup" + retrieval_strategy = ( + "batched_subject_asserted" + if grouped_hits is not None + else ("per_instance_relation_lookup") ) if grouped_hits is not None and type_count_options.include: _attach_direct_type_ids_to_fast_incidence_hits(grouped_hits) @@ -1286,16 +1351,17 @@ def _collect_type_subject_hits_for_incidence_fast( ) ) accessible_source_ids = ( - filter_accessible_concept_ids(grouped) - if enforce_access - else set(grouped) + filter_accessible_concept_ids(grouped) if enforce_access else set(grouped) ) accessible_target_ids: set[str] | None = None if enforce_access: candidate_target_ids: List[str] = [] for doc in cursor: source_id = doc.get("concept_id") - if not isinstance(source_id, str) or source_id not in accessible_source_ids: + if ( + not isinstance(source_id, str) + or source_id not in accessible_source_ids + ): continue relationships = doc.get("relationships") if not isinstance(relationships, Mapping): @@ -1303,7 +1369,9 @@ def _collect_type_subject_hits_for_incidence_fast( for predicate_id, raw_targets in relationships.items(): if not _predicate_matches_terms(predicate_id, predicate_terms): continue - candidate_target_ids.extend(_normalise_relationship_targets(raw_targets)) + candidate_target_ids.extend( + _normalise_relationship_targets(raw_targets) + ) accessible_target_ids = filter_accessible_concept_ids(candidate_target_ids) for doc in cursor: source_id = doc.get("concept_id") @@ -2894,9 +2962,7 @@ def _prime_concept_preview_cache( else set(pending_ids) ) accessible_ids = [ - concept_id - for concept_id in pending_ids - if concept_id in accessible_id_set + concept_id for concept_id in pending_ids if concept_id in accessible_id_set ] for concept_id in pending_ids: if concept_id not in accessible_id_set: @@ -3053,8 +3119,7 @@ def _filter_accessible_relationships( if isinstance(raw_targets, str): filtered[predicate_id] = ( [] - if raw_targets.startswith("#") - and raw_targets not in accessible_ids + if raw_targets.startswith("#") and raw_targets not in accessible_ids else raw_targets ) continue @@ -3544,10 +3609,7 @@ def _canonical_exact_predicate_filter_ids( tokens = _iter_predicate_filter_tokens(predicate_filter) if not tokens or any( - not token.startswith("#V#") - or "." in token - or "$" in token - or "\x00" in token + not token.startswith("#V#") or "." in token or "$" in token or "\x00" in token for token in tokens ): return [] diff --git a/src/backend/services/rag_text_relation_sync_service.py b/src/backend/services/rag_text_relation_sync_service.py index 5f41b707b..57713ab08 100644 --- a/src/backend/services/rag_text_relation_sync_service.py +++ b/src/backend/services/rag_text_relation_sync_service.py @@ -337,9 +337,6 @@ def collect_text_relation_docs_for_namespace( limit=safe_limit, ) ) - if not relations: - return [] - docs: List[TextRelationRagDoc] = [] # Small cache to avoid repeated concept lookups. @@ -421,6 +418,68 @@ def collect_text_relation_docs_for_namespace( ) ) + from src.backend.services.scoped_assertion_service import ( + list_visible_scoped_assertions, + ) + + scoped_assertions = list_visible_scoped_assertions( + subject_concept_ids=sorted(concept_allow) if concept_allow else None, + predicates=predicates, + object_kind="text", + limit=max(safe_limit, 1), + user_concept_id=user_id, + organisation_concept_id=org_id, + ) + language_allow = set(languages or ()) + for assertion in scoped_assertions: + object_text = assertion.get("object_text") + if not isinstance(object_text, dict): + continue + text = object_text.get("text") + lang = str(object_text.get("language") or "en-NZ") + subject_concept_id = str(assertion.get("subject_concept_id") or "") + predicate = str(assertion.get("predicate") or "") + assertion_id = str(assertion.get("assertion_id") or "") + if not text or not subject_concept_id or not predicate or not assertion_id: + continue + if language_allow and lang not in language_allow: + continue + if updated_since is not None: + updated_at = assertion.get("updated_at") + if isinstance(updated_at, str): + try: + if datetime.fromisoformat(updated_at) < updated_since: + continue + except ValueError: + continue + docs.append( + TextRelationRagDoc( + doc_id=f"scoped_assertion:{assertion_id}", + text=_build_rag_text( + concept_id=subject_concept_id, + predicate=predicate, + lang=lang, + text=str(text), + ), + metadata={ + "type": "scoped_knowledge_assertion", + "source": "scoped_knowledge_assertion", + "concept_id": subject_concept_id, + "subject_concept_id": subject_concept_id, + "predicate": predicate, + "assertion_id": assertion_id, + "relation_id": assertion_id, + "lang": lang, + "language": lang, + "user_id": user_id, + "organisation_concept_id": org_id, + "org_id": org_id, + "assertion_scope": assertion.get("scope"), + "canonical_publication": False, + }, + ) + ) + return docs diff --git a/src/backend/services/scoped_assertion_service.py b/src/backend/services/scoped_assertion_service.py new file mode 100644 index 000000000..a797db1a6 --- /dev/null +++ b/src/backend/services/scoped_assertion_service.py @@ -0,0 +1,381 @@ +"""Actor-scoped assertions over visible Vontology concepts. + +This store deliberately does not mutate canonical concept or text-relation +documents. It lets an authenticated user or organisation assert provenance- +bearing knowledge about a concept they can see without acquiring publication +authority over that concept. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +from typing import Any, Mapping, Sequence + +from pymongo import ReturnDocument + +from ..db.mongo_client import get_scoped_knowledge_assertions_collection +from ..security.access_control import ( + can_access_concept, + get_effective_organisation_concept_id, + get_effective_user_concept_id, +) +from .namespace_service import resolve_canonical_namespace +from .relationship_write_service import validate_predicate_concept +from .text_relation_predicate_validation_service import ( + resolve_text_relation_predicate_for_write, +) + +SCHEMA_VERSION = "scoped_knowledge_assertion.v1" +STORAGE_SURFACE = "scoped_knowledge_assertions" +SCOPE_MODES = frozenset({"user", "organisation"}) +OBJECT_KINDS = frozenset({"text", "concept"}) + + +def _clean_concept_id(value: Any, field_name: str) -> str: + token = str(value or "").strip() + if not token.startswith("#V#"): + raise ValueError(f"{field_name} must be an exact #V# concept ID") + return token + + +def _clean_optional_concept_id(value: Any) -> str | None: + token = str(value or "").strip() + return token if token.startswith("#V#") else None + + +def _normalise_evidence(value: Any) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError("evidence must be an object when provided") + # Copy through JSON so caller-owned mutable values and BSON-hostile values + # cannot leak into the persisted record. + try: + return json.loads(json.dumps(dict(value), default=str)) + except (TypeError, ValueError) as exc: + raise ValueError("evidence must be JSON-serialisable") from exc + + +def _scope_for_actor( + *, + scope_mode: str, + acting_user_concept_id: Any, + organisation_concept_id: Any, + namespace: Any, +) -> dict[str, Any]: + user_id = _clean_concept_id( + acting_user_concept_id, + "acting_user_concept_id", + ) + org_id = _clean_optional_concept_id(organisation_concept_id) + mode = str(scope_mode or "user").strip().lower() + if mode not in SCOPE_MODES: + raise ValueError("scope_mode must be 'user' or 'organisation'") + if mode == "organisation" and not org_id: + raise ValueError( + "organisation scope requires a trusted organisation concept ID" + ) + + actor_namespace = resolve_canonical_namespace( + None, + user_id, + org_id, + ) + supplied_namespace = resolve_canonical_namespace( + str(namespace or "").strip() or None, + ) + if supplied_namespace and actor_namespace and supplied_namespace != actor_namespace: + raise ValueError( + "namespace does not match the trusted user and organisation scope" + ) + audience_key = f"user:{user_id}" if mode == "user" else f"org:{org_id}" + return { + "mode": mode, + "user_concept_id": user_id, + "organisation_concept_id": org_id, + "namespace": actor_namespace, + "audience_keys": [audience_key], + } + + +def _assertion_identity(identity: Mapping[str, Any]) -> str: + encoded = json.dumps(identity, sort_keys=True, separators=(",", ":")) + return f"ska_{hashlib.sha256(encoded.encode('utf-8')).hexdigest()[:32]}" + + +def _serialise(document: Mapping[str, Any] | None) -> dict[str, Any] | None: + if not isinstance(document, Mapping): + return None + result = {key: value for key, value in document.items() if key != "_id"} + for field in ("created_at", "updated_at"): + value = result.get(field) + if isinstance(value, datetime): + result[field] = value.isoformat() + return result + + +def upsert_scoped_assertion( + *, + subject_concept_id: Any, + predicate: Any, + target_text: Any = None, + target_concept_id: Any = None, + language: Any = "en-NZ", + scope_mode: str = "user", + evidence: Any = None, + acting_user_concept_id: Any, + organisation_concept_id: Any = None, + namespace: Any = None, + turn_id: Any = None, + canonical_publication: bool = False, +) -> dict[str, Any]: + """Upsert one idempotent scoped assertion and return canonical read-back.""" + + if canonical_publication is not False: + raise ValueError("scoped assertions cannot request canonical publication") + subject_id = _clean_concept_id(subject_concept_id, "subject_concept_id") + if not can_access_concept(subject_id): + raise PermissionError("subject concept is not visible to the current actor") + + has_text = isinstance(target_text, str) and bool(target_text.strip()) + has_concept = isinstance(target_concept_id, str) and bool(target_concept_id.strip()) + if has_text == has_concept: + raise ValueError("provide exactly one of target_text or target_concept_id") + + if has_text: + resolution = resolve_text_relation_predicate_for_write(predicate) + storage_predicate = resolution.storage_predicate + object_kind = "text" + object_text = str(target_text).strip() + language_token = str(language or "en-NZ").strip() or "en-NZ" + object_concept_id = None + identity_object: dict[str, Any] = { + "text": object_text, + "language": language_token, + } + else: + storage_predicate = _clean_concept_id(predicate, "predicate") + is_valid, error_code, error_details = validate_predicate_concept( + storage_predicate + ) + if not is_valid: + raise ValueError( + f"{error_code or 'invalid_predicate'}: {error_details or {}}" + ) + object_kind = "concept" + object_text = None + language_token = None + object_concept_id = _clean_concept_id( + target_concept_id, + "target_concept_id", + ) + if not can_access_concept(object_concept_id): + raise PermissionError("target concept is not visible to the current actor") + identity_object = {"concept_id": object_concept_id} + + scope = _scope_for_actor( + scope_mode=scope_mode, + acting_user_concept_id=acting_user_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + ) + identity = { + "scope_mode": scope["mode"], + "audience_keys": scope["audience_keys"], + "subject_concept_id": subject_id, + "predicate": storage_predicate, + "object_kind": object_kind, + "object": identity_object, + } + assertion_id = _assertion_identity(identity) + now = datetime.now(timezone.utc) + provenance = { + "asserted_by_user_concept_id": scope["user_concept_id"], + "organisation_concept_id": scope["organisation_concept_id"], + "namespace": scope["namespace"], + "turn_id": str(turn_id or "").strip() or None, + "capability_name": "upsert_scoped_assertion", + "evidence": _normalise_evidence(evidence), + } + update_fields: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "assertion_id": assertion_id, + "subject_concept_id": subject_id, + "predicate": storage_predicate, + "object_kind": object_kind, + "scope": scope, + "provenance": provenance, + "canonical_publication": False, + "status": "asserted", + "updated_at": now, + } + if object_kind == "text": + update_fields["object_text"] = { + "text": object_text, + "language": language_token, + } + update_fields["object_concept_id"] = None + else: + update_fields["object_concept_id"] = object_concept_id + update_fields["object_text"] = None + + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + raise RuntimeError("scoped assertion storage is unavailable") + existing = collection.find_one({"assertion_id": assertion_id}, {"_id": 1}) + persisted = collection.find_one_and_update( + {"assertion_id": assertion_id}, + { + "$set": update_fields, + "$setOnInsert": {"created_at": now}, + }, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + read_back = _serialise(persisted) + if read_back is None: + raise RuntimeError("scoped assertion write did not produce read-back") + return { + "success": True, + "effect_status": "succeeded", + "changed": existing is None, + "assertion_id": assertion_id, + "assertion": read_back, + "canonical_read_back": read_back, + "canonical_publication": False, + "storage_surface": STORAGE_SURFACE, + } + + +def _audience_keys_for_actor( + *, + user_concept_id: str | None = None, + organisation_concept_id: str | None = None, +) -> list[str]: + user_id = _clean_optional_concept_id( + user_concept_id or get_effective_user_concept_id() + ) + org_id = _clean_optional_concept_id( + organisation_concept_id or get_effective_organisation_concept_id() + ) + keys: list[str] = [] + if user_id: + keys.append(f"user:{user_id}") + if org_id: + keys.append(f"org:{org_id}") + return keys + + +def list_visible_scoped_assertions( + *, + subject_concept_ids: Sequence[str] | None = None, + argument_concept_id: str | None = None, + predicates: Sequence[str] | None = None, + object_kind: str | None = None, + limit: int = 200, + user_concept_id: str | None = None, + organisation_concept_id: str | None = None, +) -> list[dict[str, Any]]: + """Return scoped assertions visible to the effective or explicit actor.""" + + audience_keys = _audience_keys_for_actor( + user_concept_id=user_concept_id, + organisation_concept_id=organisation_concept_id, + ) + if not audience_keys: + return [] + query: dict[str, Any] = { + "scope.audience_keys": {"$in": audience_keys}, + "status": "asserted", + } + if subject_concept_ids: + visible_subjects = sorted( + { + str(item).strip() + for item in subject_concept_ids + if isinstance(item, str) + and item.strip() + and can_access_concept(item.strip()) + } + ) + if not visible_subjects: + return [] + query["subject_concept_id"] = {"$in": visible_subjects} + if argument_concept_id: + concept_id = _clean_concept_id(argument_concept_id, "argument_concept_id") + if not can_access_concept(concept_id): + return [] + query["$or"] = [ + {"subject_concept_id": concept_id}, + {"object_concept_id": concept_id}, + ] + if predicates: + predicate_values = sorted( + { + str(item).strip() + for item in predicates + if isinstance(item, str) and item.strip() + } + ) + if predicate_values: + query["predicate"] = {"$in": predicate_values} + if object_kind is not None: + if object_kind not in OBJECT_KINDS: + raise ValueError("object_kind must be 'text' or 'concept'") + query["object_kind"] = object_kind + + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + return [] + cursor = ( + collection.find(query) + .sort([("updated_at", -1), ("assertion_id", 1)]) + .limit(max(1, min(int(limit), 1000))) + ) + return [ + serialised + for document in cursor + if (serialised := _serialise(document)) is not None + ] + + +def scoped_text_assertion_to_relation_row( + assertion: Mapping[str, Any], +) -> dict[str, Any] | None: + """Project a scoped text assertion into the generic text-read row shape.""" + + object_text = assertion.get("object_text") + if assertion.get("object_kind") != "text" or not isinstance(object_text, Mapping): + return None + assertion_id = str(assertion.get("assertion_id") or "").strip() + return { + "subject_concept_id": assertion.get("subject_concept_id"), + "text": object_text.get("text"), + "lang": object_text.get("language"), + "text_value_id": None, + "predicate": assertion.get("predicate"), + "relation_id": assertion_id or None, + "assertion_id": assertion_id or None, + "context": { + "assertion_scope": assertion.get("scope"), + "canonical_publication": False, + "storage_surface": STORAGE_SURFACE, + }, + "provenance": assertion.get("provenance") or {}, + "assertion_scope": assertion.get("scope"), + "canonical_publication": False, + "storage_surface": STORAGE_SURFACE, + "text_value_created_at": assertion.get("created_at"), + "text_value_updated_at": assertion.get("updated_at"), + "relation_created_at": assertion.get("created_at"), + "relation_updated_at": assertion.get("updated_at"), + } + + +__all__ = [ + "list_visible_scoped_assertions", + "scoped_text_assertion_to_relation_row", + "upsert_scoped_assertion", +] diff --git a/src/backend/services/text_value_service.py b/src/backend/services/text_value_service.py index f3c6e834e..6e237591c 100644 --- a/src/backend/services/text_value_service.py +++ b/src/backend/services/text_value_service.py @@ -503,15 +503,51 @@ def get_texts_for_concepts( rows_by_concept: Dict[str, List[Dict[str, Any]]] = { subject_id: [] for subject_id in ordered_subject_ids } - if not relations: - return rows_by_concept + rows = ( + _resolve_text_rows_from_relations( + relations, + lang=lang, + limit=None, + max_time_ms=max_time_ms, + ) + if relations + else [] + ) + from .scoped_assertion_service import ( + list_visible_scoped_assertions, + scoped_text_assertion_to_relation_row, + ) - rows = _resolve_text_rows_from_relations( - relations, - lang=lang, - limit=None, - max_time_ms=max_time_ms, + scoped_assertions = list_visible_scoped_assertions( + subject_concept_ids=ordered_subject_ids, + predicates=( + [predicate] + if predicate + else [ + str(item).strip() + for item in predicates or () + if isinstance(item, str) and str(item).strip() + ] + or None + ), + object_kind="text", + limit=max(len(ordered_subject_ids) * max(limit_per_concept, 1), 1), ) + for assertion in scoped_assertions: + scoped_row = scoped_text_assertion_to_relation_row(assertion) + if scoped_row is None: + continue + if lang and scoped_row.get("lang") != lang: + continue + rows.append(scoped_row) + + if recent_first: + rows.sort( + key=lambda row: str( + row.get("relation_updated_at") or row.get("relation_created_at") or "" + ), + reverse=True, + ) for row in rows: subject_concept_id = row.get("subject_concept_id") if not isinstance(subject_concept_id, str): @@ -716,42 +752,23 @@ def get_preferred_texts_for_concepts( if not ordered_subject_ids: return {} - relation_filter: Dict[str, Any] = { - "subject_concept_id": {"$in": ordered_subject_ids} - } normalised_precedence = _normalise_predicate_precedence(predicate_precedence) - if normalised_precedence: - allowed_predicates = sorted( + allowed_predicates = ( + sorted( { predicate for predicate_group in normalised_precedence for predicate in predicate_group } ) - relation_filter["predicate"] = {"$in": allowed_predicates} - - relation_limit = max(len(ordered_subject_ids) * max(limit_per_concept, 1), 1) - relations = list( - TextRelationsRepository.find(relation_filter, limit=relation_limit) + if normalised_precedence + else None + ) + rows_by_subject = get_texts_for_concepts( + ordered_subject_ids, + predicates=allowed_predicates, + limit_per_concept=limit_per_concept, ) - if not relations: - return {} - - rows = _resolve_text_rows_from_relations(relations, limit=None) - if not rows: - return {} - - rows_by_subject: Dict[str, List[Dict[str, Any]]] = { - subject_id: [] for subject_id in ordered_subject_ids - } - for row in rows: - subject_concept_id = row.get("subject_concept_id") - if not isinstance(subject_concept_id, str): - continue - subject_rows = rows_by_subject.get(subject_concept_id) - if subject_rows is None or len(subject_rows) >= limit_per_concept: - continue - subject_rows.append(row) preferred_rows: Dict[str, Dict[str, Any]] = {} for subject_concept_id, subject_rows in rows_by_subject.items(): @@ -1150,6 +1167,47 @@ def get_text_relations_summary( bucket["latest_updated_at"] = updated_at bucket["latest_relation_id"] = rel_id + from .scoped_assertion_service import list_visible_scoped_assertions + + scoped_assertions = list_visible_scoped_assertions( + subject_concept_ids=[subject_concept_id], + predicates=predicates, + object_kind="text", + limit=1000, + ) + for assertion in scoped_assertions: + object_text = assertion.get("object_text") + if not isinstance(object_text, dict): + continue + predicate = str(assertion.get("predicate") or "") + lang = str(object_text.get("language") or "") + if language_allow is not None and lang not in language_allow: + continue + key = (predicate, lang) + bucket = grouped.get(key) + if bucket is None: + bucket = { + "predicate": predicate, + "language": lang, + "count": 0, + "relation_ids": [], + "latest_relation_id": None, + "latest_updated_at": None, + } + grouped[key] = bucket + bucket["count"] += 1 + bucket["scoped_count"] = int(bucket.get("scoped_count") or 0) + 1 + assertion_id = str(assertion.get("assertion_id") or "") + if assertion_id and len(bucket["relation_ids"]) < max_relation_ids_per_group: + bucket["relation_ids"].append(assertion_id) + updated_at = assertion.get("updated_at") or assertion.get("created_at") + current_latest = bucket.get("latest_updated_at") + if updated_at is not None and ( + current_latest is None or str(updated_at) > str(current_latest) + ): + bucket["latest_updated_at"] = updated_at + bucket["latest_relation_id"] = assertion_id + summary_items = list(grouped.values()) summary_items.sort( key=lambda x: (x.get("predicate") or "", x.get("language") or "") @@ -1165,7 +1223,8 @@ def get_text_relations_summary( "concept_id": subject_concept_id, "groups": summary_items, "groups_found": len(summary_items), - "total_relations_scanned": len(rels), + "total_relations_scanned": len(rels) + len(scoped_assertions), + "scoped_relations_scanned": len(scoped_assertions), "max_relation_ids_per_group": max_relation_ids_per_group, } diff --git a/tests/backend/test_internal_mcp_catalogue_builds.py b/tests/backend/test_internal_mcp_catalogue_builds.py index cb6e44112..8184101b7 100644 --- a/tests/backend/test_internal_mcp_catalogue_builds.py +++ b/tests/backend/test_internal_mcp_catalogue_builds.py @@ -5,6 +5,8 @@ def test_internal_mcp_catalogue_builds_and_includes_relationship_tools(): methods = set(catalogue.list_methods()) assert "add_relationship" in methods + assert "upsert_scoped_assertion" in methods + assert "list_scoped_assertions" in methods assert "remove_relationship" in methods assert "preview_remove_relationship" in methods assert "remove_relationships_bulk" in methods @@ -112,13 +114,13 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): "gmail_profile": "represented-profile", "turn_namespace": "#V#ordinary_actor@ordinary_org", "actor_user_concept_id": "#V#ordinary_actor", + "actor_organisation_concept_id": "#V#ordinary_org", + "turn_id": "turn-1", }, ) ) delegated_effects = { - name - for name in actor_mail_reads - if catalogue.get(name).category == "write" + name for name in actor_mail_reads if catalogue.get(name).category == "write" } # A newly registered ordinary read does not need a second positive @@ -145,6 +147,7 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): }.isdisjoint(actor_mail_reads) assert delegated_effects == { "create_concepts", + "upsert_scoped_assertion", "upsert_text_relation", "add_relationship", } @@ -166,6 +169,17 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): assert catalogue.get("upsert_text_relation").ordinary_turn_fixed_arguments == { "provenance": None, } + scoped_definition = catalogue.get("upsert_scoped_assertion") + assert scoped_definition.ordinary_turn_trusted_argument_bindings == { + "acting_user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + "namespace": "turn_namespace", + "turn_id": "turn_id", + } + assert scoped_definition.ordinary_turn_fixed_arguments == { + "canonical_publication": False, + } + assert scoped_definition.ordinary_turn_mutation_subject_argument is None # Server-bound resources are absent until the entry point supplies the # actor's represented binding; the model never selects the profile. @@ -213,6 +227,7 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): "turn_execution_build_selector_benchmark", } <= actor_reads + def test_actor_scoped_private_reads_reject_payload_only_identity(monkeypatch): from src.backend.integrations.internal_mcp import ( InternalMCPGateway, @@ -306,10 +321,7 @@ def test_failure_case_learning_reads_require_operator_provenance(): ): result = gateway.invoke(method_name, {}).payload assert result["success"] is False - assert ( - result["error_code"] - == "workflow_global_admin_authority_required" - ) + assert result["error_code"] == "workflow_global_admin_authority_required" def test_internal_mcp_gmail_list_messages_accepts_max_results_aliases(): diff --git a/tests/backend/test_mongo_collection_index_guards.py b/tests/backend/test_mongo_collection_index_guards.py index 8a7603340..230d650fa 100644 --- a/tests/backend/test_mongo_collection_index_guards.py +++ b/tests/backend/test_mongo_collection_index_guards.py @@ -71,6 +71,33 @@ def test_text_value_indexes_are_ensured_once_per_database(monkeypatch): } +def test_scoped_assertion_indexes_are_ensured_once_per_database(monkeypatch): + _reset_collection_index_state(monkeypatch) + db = _FakeDb("test_von_db", client=object()) + monkeypatch.setattr(mc, "get_db", lambda: db) + + coll_first = cast( + _FakeCollection, + mc.get_scoped_knowledge_assertions_collection(), + ) + coll_second = cast( + _FakeCollection, + mc.get_scoped_knowledge_assertions_collection(), + ) + + assert coll_first is coll_second + assert coll_first is not None + assert len(coll_first.create_index_calls) == 4 + assert { + str(call["kwargs"].get("name")) for call in coll_first.create_index_calls + } == { + "assertion_id_unique", + "subject_audience_updated_at_desc", + "object_audience_updated_at_desc", + "predicate_audience_updated_at_desc", + } + + def test_interaction_session_indexes_are_ensured_once_per_database(monkeypatch): _reset_collection_index_state(monkeypatch) db = _FakeDb("test_von_db", client=object()) @@ -106,9 +133,7 @@ def test_application_settings_index_is_ensured_once_per_database(monkeypatch): assert coll_first is coll_second assert coll_first is not None assert len(coll_first.create_index_calls) == 1 - assert coll_first.create_index_calls[0]["keys"] == [ - ("setting_name", mc.ASCENDING) - ] + assert coll_first.create_index_calls[0]["keys"] == [("setting_name", mc.ASCENDING)] assert coll_first.create_index_calls[0]["kwargs"] == {"unique": True} @@ -180,8 +205,7 @@ def test_text_relation_indexes_cover_concept_search_and_atlas_name_lookup(monkey assert coll is not None calls_by_name = { - str(call["kwargs"].get("name") or ""): call - for call in coll.create_index_calls + str(call["kwargs"].get("name") or ""): call for call in coll.create_index_calls } assert calls_by_name.keys() >= { "context_name_type_predicate_text", @@ -212,9 +236,7 @@ def test_relationship_extent_indexes_add_target_first_lookup_without_dropping_dr assert list(indexes["target_predicate_source_lookup"]["key"].items()) == ( wrong_keys ) - assert list( - indexes["target_value_predicate_source_lookup_v2"]["key"].items() - ) == [ + assert list(indexes["target_value_predicate_source_lookup_v2"]["key"].items()) == [ ("target_value", mc.ASCENDING), ("predicate_id", mc.ASCENDING), ("source_concept_id", mc.ASCENDING), @@ -286,9 +308,7 @@ def test_relationship_extent_index_ensure_does_not_replace_named_unique_drift(): mc._ensure_relationship_extent_index_indexes(coll) - index = { - item["name"]: item for item in coll.list_indexes() - }["relation_id_1_unique"] + index = {item["name"]: item for item in coll.list_indexes()}["relation_id_1_unique"] assert index.get("unique") is not True diff --git a/tests/backend/test_rag_text_relation_sync_service.py b/tests/backend/test_rag_text_relation_sync_service.py index 40581406e..2c3388390 100644 --- a/tests/backend/test_rag_text_relation_sync_service.py +++ b/tests/backend/test_rag_text_relation_sync_service.py @@ -3,7 +3,6 @@ from typing import Any, Dict, Optional - class _ConceptsCollection: def __init__(self, visible_concepts: set[str]): self.visible_concepts = set(visible_concepts) @@ -109,3 +108,46 @@ def _tv_find_one(filter_doc, _projection=None): assert meta["user_id"] == "#V#user" assert meta["organisation_concept_id"] == "#V#org" assert meta["org_id"] == "#V#org" + + +def test_collect_includes_scoped_assertion_for_namespace(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + from src.backend.services import scoped_assertion_service + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + scoped_assertion_service, + "list_visible_scoped_assertions", + lambda **_kwargs: [ + { + "assertion_id": "ska_123", + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": "Organisation-scoped programme context.", + "language": "en-NZ", + }, + "scope": { + "mode": "organisation", + "audience_keys": ["org:#V#org"], + }, + "canonical_publication": False, + } + ], + ) + + docs = svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + ) + + assert len(docs) == 1 + assert docs[0].doc_id == "scoped_assertion:ska_123" + assert "Organisation-scoped programme context." in docs[0].text + assert docs[0].metadata["type"] == "scoped_knowledge_assertion" + assert docs[0].metadata["canonical_publication"] is False + assert docs[0].metadata["organisation_concept_id"] == "#V#org" diff --git a/tests/backend/test_scoped_assertion_service.py b/tests/backend/test_scoped_assertion_service.py new file mode 100644 index 000000000..5ca7e1b06 --- /dev/null +++ b/tests/backend/test_scoped_assertion_service.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import mongomock +import pytest + + +def _collection(): + return mongomock.MongoClient()["von_test"]["scoped_knowledge_assertions"] + + +def test_visible_global_subject_accepts_user_scoped_text_assertion(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + + first = service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Gillian leads our research programme.", + scope_mode="organisation", + evidence={"source": "user_instruction"}, + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#university_of_auckland_strong_ai_lab", + namespace="#V#michael_witbrock@university_of_auckland_strong_ai_lab", + turn_id="turn-1", + canonical_publication=False, + ) + second = service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Gillian leads our research programme.", + scope_mode="organisation", + evidence={"source": "user_instruction"}, + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#university_of_auckland_strong_ai_lab", + namespace="#V#michael_witbrock@university_of_auckland_strong_ai_lab", + turn_id="turn-1", + canonical_publication=False, + ) + + assert first["success"] is True + assert first["changed"] is True + assert second["changed"] is False + assert first["assertion_id"] == second["assertion_id"] + assert collection.count_documents({}) == 1 + read_back = first["canonical_read_back"] + assert read_back["subject_concept_id"] == "#V#gillian_dobbie" + assert read_back["scope"]["mode"] == "organisation" + assert read_back["canonical_publication"] is False + assert read_back["provenance"]["turn_id"] == "turn-1" + + +def test_scoped_assertions_do_not_cross_actor_or_organisation(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Organisation-private knowledge.", + scope_mode="organisation", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#organisation_a", + namespace="#V#michael_witbrock@organisation_a", + canonical_publication=False, + ) + + same_org = service.list_visible_scoped_assertions( + subject_concept_ids=["#V#gillian_dobbie"], + user_concept_id="#V#someone_else", + organisation_concept_id="#V#organisation_a", + ) + other_org = service.list_visible_scoped_assertions( + subject_concept_ids=["#V#gillian_dobbie"], + user_concept_id="#V#someone_else", + organisation_concept_id="#V#organisation_b", + ) + + assert len(same_org) == 1 + assert other_org == [] + + +def test_service_rejects_namespace_that_disagrees_with_trusted_actor(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + _collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + + with pytest.raises(ValueError, match="namespace does not match"): + service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Attempted scope substitution.", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#attacker@other_org", + canonical_publication=False, + ) + + +def test_concept_target_assertion_is_retrievable_from_either_argument(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + monkeypatch.setattr( + service, + "validate_predicate_concept", + lambda _predicate: (True, None, None), + ) + receipt = service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="#V#co_directs", + target_concept_id="#V#centre_for_machine_learning_for_social_good", + scope_mode="user", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + + from_subject = service.list_visible_scoped_assertions( + argument_concept_id="#V#gillian_dobbie", + object_kind="concept", + user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + ) + from_target = service.list_visible_scoped_assertions( + argument_concept_id="#V#centre_for_machine_learning_for_social_good", + object_kind="concept", + user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + ) + + assert receipt["success"] is True + assert [item["assertion_id"] for item in from_subject] == [receipt["assertion_id"]] + assert [item["assertion_id"] for item in from_target] == [receipt["assertion_id"]] + + +def test_trusted_tool_payload_replaces_model_supplied_scope(): + from src.backend.integrations.internal_mcp import ( + InternalMCPGateway, + InternalMCPTransport, + build_default_catalogue, + ) + from src.backend.services.adaptive_turn_service import _trusted_tool_payload + + gateway = InternalMCPGateway( + catalogue=build_default_catalogue(), + transport=InternalMCPTransport(), + enabled=True, + ) + payload = _trusted_tool_payload( + gateway=gateway, + tool_name="upsert_scoped_assertion", + model_payload={ + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "target_text": "Scoped fact", + "acting_user_concept_id": "#V#attacker", + "organisation_concept_id": "#V#other_org", + "namespace": "#V#attacker@other_org", + "turn_id": "forged-turn", + "canonical_publication": True, + }, + trusted_argument_values={ + "actor_user_concept_id": "#V#michael_witbrock", + "actor_organisation_concept_id": "#V#trusted_org", + "turn_namespace": "#V#michael_witbrock@trusted_org", + "turn_id": "trusted-turn", + }, + ) + + assert payload["acting_user_concept_id"] == "#V#michael_witbrock" + assert payload["organisation_concept_id"] == "#V#trusted_org" + assert payload["namespace"] == "#V#michael_witbrock@trusted_org" + assert payload["turn_id"] == "trusted-turn" + assert payload["canonical_publication"] is False + + +def test_generic_text_read_merges_visible_scoped_assertions(monkeypatch): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service as scoped_service + from src.backend.services import text_value_service + + collection = _collection() + monkeypatch.setattr( + scoped_service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + scoped_service, + "can_access_concept", + lambda _concept_id: True, + ) + monkeypatch.setattr( + text_value_service.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + scoped_service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Scoped programme context.", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + + with override_current_actor("#V#michael_witbrock", "#V#trusted_org"): + rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + ) + + assert len(rows) == 1 + assert rows[0]["text"] == "Scoped programme context." + assert rows[0]["assertion_id"].startswith("ska_") + assert rows[0]["canonical_publication"] is False + assert rows[0]["storage_surface"] == "scoped_knowledge_assertions" From bfc31d8cc08d2f6613ec9b7e9cef75a2d8c007e7 Mon Sep 17 00:00:00 2001 From: witbrock Date: Wed, 29 Jul 2026 15:28:06 +0100 Subject: [PATCH 2/2] JVNAUTOSCI-2612: complete scoped knowledge recovery Add explicit base and actor-effective views, trusted scoped assertion authority, bounded RAG projection and retraction, cold-agent semantic-coherence guidance, and proportional scale/security validation. --- AGENTS.md | 12 +- docs/design_index.md | 4 +- .../contextual_knowledge_evolution.md | 201 +++ src/backend/db/mongo_client.py | 68 +- .../integrations/internal_mcp/catalogue.py | 1095 ++++++++++++--- src/backend/mcp_server/mcp_stdio_server.py | 266 ++-- src/backend/mcp_server/vontology_mcp.json | 53 +- src/backend/security/access_control.py | 19 + src/backend/services/adaptive_turn_service.py | 252 +++- .../services/concept_relation_service.py | 110 +- .../rag_backends/llamaindex_backend.py | 159 ++- .../rag_text_relation_change_hook_service.py | 222 ++- .../rag_text_relation_sync_service.py | 1229 +++++++++++++---- .../services/scoped_assertion_service.py | 1029 ++++++++++++-- .../services/scoped_rag_authority_service.py | 242 ++++ .../services/testing_theory_service.py | 7 +- ...t_relation_predicate_validation_service.py | 14 + src/backend/services/text_value_service.py | 286 +++- src/utilities/reindex_text_relations.py | 112 +- tests/backend/test_adaptive_turn_service.py | 333 ++++- .../test_context_semantic_coherence.py | 853 ++++++++++++ .../test_description_metadata_preservation.py | 6 + .../test_internal_mcp_catalogue_builds.py | 31 + ...est_internal_mcp_concept_exists_gateway.py | 9 +- ...internal_mcp_scoped_assertion_authority.py | 820 +++++++++++ .../test_mongo_collection_index_guards.py | 85 +- tests/backend/test_rag_provenance_contract.py | 144 ++ tests/backend/test_rag_retrieval_state.py | 220 ++- tests/backend/test_rag_service.py | 245 ++++ ...t_rag_text_relation_change_hook_service.py | 283 ++++ .../test_rag_text_relation_sync_service.py | 1050 +++++++++++++- .../test_rag_text_relations_mcp_read_tools.py | 35 +- .../test_reindex_text_relations_cli.py | 167 ++- .../backend/test_scoped_assertion_service.py | 764 +++++++++- 34 files changed, 9481 insertions(+), 944 deletions(-) create mode 100644 docs/engineering/contextual_knowledge_evolution.md create mode 100644 src/backend/services/scoped_rag_authority_service.py create mode 100644 tests/backend/test_context_semantic_coherence.py create mode 100644 tests/backend/test_internal_mcp_scoped_assertion_authority.py create mode 100644 tests/backend/test_rag_text_relation_change_hook_service.py diff --git a/AGENTS.md b/AGENTS.md index 0ae6e9c8f..a75b0fd39 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ - **Lifecycle:** Active - **Authority:** Governing instructions for work in this repository, subordinate to current explicit user direction and higher-level safety rules -- **Last reviewed:** 27 July 2026 +- **Last reviewed:** 29 July 2026 - **Review trigger:** A material change to Von's product focus, authority model, security posture, or acceptance doctrine @@ -150,6 +150,16 @@ never be forced onto newer evidence. latency, cost, human burden, failure recovery, and maintenance impact. 14. **Prefer subtraction.** Remove obsolete stages, fallbacks, prompts, tools, tests, and documentation when evidence shows they add cost without value. +15. **Preserve contextual degrees of freedom.** When work changes durable + assertions or their retrieval, do not silently make concept identity, + assertion context, provenance, audience or authority, publication status, + and physical storage interchangeable. A bounded implementation may use an + implicit context or a separate store; keep that simplification explicit and + local enough that a later context or theory mechanism need not reinterpret + domain meaning. Add general machinery only when a current capability or + evidence needs it. Use the + [contextual knowledge evolution guide](docs/engineering/contextual_knowledge_evolution.md) + when this boundary is material. ## 4. Choosing the authority surface diff --git a/docs/design_index.md b/docs/design_index.md index 695bcca44..495ccd6ad 100644 --- a/docs/design_index.md +++ b/docs/design_index.md @@ -6,7 +6,7 @@ override current user direction, `AGENTS.md`, live represented authority, or live evidence - **Owner:** Von maintainers -- **Last reviewed:** 25 July 2026 +- **Last reviewed:** 29 July 2026 - **Review trigger:** Any change to `AGENTS.md` reading routes, canonical document selection, or document supersession - **Scope:** Tracked design, engineering, operational, review, and generated @@ -85,6 +85,7 @@ live. | Workflow or orchestration | Relevant vocabulary/semantics in the [VWL manual](engineering/von_workflow_language_manual.md); domain examples and appendices are reference material | | Prompts, models, routing, optimisation, or fine-tuning | [prompt programmes and model routing](engineering/prompt_programs_and_model_routing_playbook.md) | | Retrieval, memory, RAG, KB growth, or long-horizon state | [agent memory and enduring knowledge](engineering/agent_memory_and_enduring_knowledge.md) | +| Durable assertions, context-sensitive retrieval, hypotheses, publication or promotion, or user-, organisation-, project-, source-, theory-, or time-relative knowledge | [contextual knowledge evolution](engineering/contextual_knowledge_evolution.md) | | Evaluation, benchmarks, or research-sensitive architecture | [agent evaluation and research uptake](engineering/agent_evaluation_and_research_uptake.md) | | Minimal imposition, elicitation, or write policy | [minimal-imposition principle](engineering/minimal_imposition_design_principle.md) | | Frontend or browser acceptance | [Frontend browser validation](engineering/frontend_browser_user_view_validation.md), at the validation tier justified by the claim | @@ -106,6 +107,7 @@ from `AGENTS.md`. | [VWL manual](engineering/von_workflow_language_manual.md) | Active manual | Canonical reference for VWL vocabulary and runtime interfaces; consult relevant sections, and verify live artefacts/behaviour | | [Prompt programmes and model routing](engineering/prompt_programs_and_model_routing_playbook.md) | Active playbook | Normative only within its stated prompt/model scope | | [Agent memory and enduring knowledge](engineering/agent_memory_and_enduring_knowledge.md) | Active design guide | Normative only within its stated memory/retrieval scope | +| [Contextual knowledge evolution](engineering/contextual_knowledge_evolution.md) | Active design guide | Advisory under `AGENTS.md` when assertion/context distinctions are material; it does not select a microtheory formalism or prove present implementation | | [Agent evaluation and research uptake](engineering/agent_evaluation_and_research_uptake.md) | Active protocol/design guide | Normative only for the evaluation or research claim being made | | [Minimal-imposition principle](engineering/minimal_imposition_design_principle.md) | Active principle | Explanatory guidance under `AGENTS.md`; it adds no independent per-task gates | | [Frontend browser validation](engineering/frontend_browser_user_view_validation.md) | Active practical guide | Risk-tiered reference; browser evidence needs a dated locator when used | diff --git a/docs/engineering/contextual_knowledge_evolution.md b/docs/engineering/contextual_knowledge_evolution.md new file mode 100644 index 000000000..5ce491af3 --- /dev/null +++ b/docs/engineering/contextual_knowledge_evolution.md @@ -0,0 +1,201 @@ +# Contextual Knowledge Evolution + +- **Kind:** Design guide +- **Lifecycle:** Active +- **Authority:** Advisory under the contextual-degrees-of-freedom invariant in + [`AGENTS.md`](../../AGENTS.md) +- **Authority scope:** Durable assertions, context-sensitive retrieval, + hypotheses, knowledge publication or promotion, and actor-, organisation-, + project-, source-, theory-, or time-relative knowledge +- **Owner:** Von maintainers +- **Last reviewed:** 29 July 2026 +- **Review trigger:** Adoption of a first-class context/theory model, material + multi-tenant scale evidence, material changes to the named implementation + surfaces, or a change to assertion, namespace, inclusion, or publication + semantics +- **Live implementation evidence:** Revalidate the services named in section 4; + the map is a 29 July 2026 observation, not a permanent architecture contract + +## 1. When to use this guide + +Use it when a change does one or more of the following: + +- creates or changes durable facts, claims, hypotheses, uncertain relations, or + their lifecycle; +- combines knowledge sources during a read; +- introduces or changes context, theory, dossier, inclusion, inheritance, + promotion, publication, endorsement, or conflict semantics; +- makes user, organisation, project, document, workflow, experiment, or time + scope part of assertion meaning; or +- changes what “canonical”, “global”, “shared”, or “visible” means for + represented knowledge. + +Do not use it for an ordinary UI, transport, telemetry, cache, raw document, or +transactional-data change that leaves represented meaning unchanged. + +This is a trigger for a small design check, not an architecture approval gate. + +## 2. Direction without a selected context architecture + +Von should preserve these distinctions when they matter to the current user +job: + +| Dimension | Question | +|---|---| +| Concept identity | Which entity or predicate is being discussed? | +| Assertion context | Under which assumptions, theory, situation, or viewpoint does the assertion hold? | +| Provenance | Who or what supplied the assertion, through which evidence and process? | +| Audience and authority | Who may discover, read, change, endorse, or publish it? | +| Publication or endorsement | Has a person or group adopted it into a named shared context? | +| Lifecycle and confidence | Is it proposed, active, contradicted, retracted, expired, or uncertain? | +| Physical storage | Which collection, graph, document, or index currently holds it? | + +The dimensions need not become seven fields on every record. A bounded +capability may use an implicit context, a fixed audience, or a separate store. +Keep that simplifying assumption explicit and local enough that a later +mechanism can replace it without reinterpreting domain meaning. + +In particular: + +- stable Vontology concept identity does not imply context-free truth; +- a shared or base publication context is a context, not “the one true theory”; +- semantic context inclusion must not grant access or reveal a private + context's existence; +- visibility does not imply endorsement, and provenance does not imply truth; +- a collection boundary may implement a distinction but does not define its + semantics; and +- microtheories, named graphs, assertion deltas, embedded theory slices, and + separate stores remain candidate mechanisms, not doctrine. + +Qualify “canonical” when using it. Say canonical **concept identity**, live +**authority**, base **publication**, or durable **read-back**, as applicable. + +## 3. Four questions for a coding agent + +For a triggered change, ask only: + +1. What user outcome is being improved, and what is the generic semantic + failure behind the immediate symptom? +2. Which current assertion, context, theory, provenance, and publication + surfaces already carry the relevant meaning? +3. Which distinctions in section 2 matter now, and which are deliberately + simplified? +4. What is the smallest seam that preserves today's job, and what evidence or + capability should trigger reconsideration? + +Reusing, extending, adapting, or temporarily diverging from a current mechanism +are possible answers, not required labels. When material, record the semantic +distinction being simplified, the existing mechanism or seam used, and the +evidence or capability that should trigger reconsideration. Omit anything +irrelevant. + +Do not generalise merely because a general future is imaginable. Conversely, +do not let the nearest collection, namespace, or acceptance test silently +decide eventual semantics. Test assertion meaning at the boundary making the +claim. + +## 4. Current starting points — verify live + +As observed on 29 July 2026, related capabilities are distributed across: + +- base concept and text relations in `concept_service.py`, + `concept_relation_service.py`, and `text_value_service.py`; +- actor- and organisation-visible assertion deltas in + `scoped_assertion_service.py`; +- local assertions, lifecycle, diff, promotion, rollback, and stored inclusion + identifiers in `testing_theory_service.py`; +- evidence, hypotheses, branches, and theory references in + `context_bundle_service.py`; and +- uncertainty and promotion state in `uncertain_relationship_service.py`. + +These are working surfaces, not proof of five distinct domain ontologies. Before +adding a sibling service or schema, inspect whether the job can reuse or adapt +one of them. + +Material current limitations include: + +- the scoped-assertion audience currently doubles as an implicit context; +- Testing Theory inclusion identifiers do not yet constitute a general + inherited query model; +- storage-specific metadata is exposed by compatibility adapters; and +- no common conflict, precedence, lifting, or context-selection semantics have + been selected. + +## 5. Coding, testing, and Jira consequences + +Reusable reads should expose the semantic view and lineage a caller needs, +rather than making collection order the public contract. Preserve established +generic contracts where possible, and select any actor-relative view at a +trusted authority boundary. + +Policy-neutral Python may provide bounded persistence, retrieval, lineage, +validation, access enforcement, telemetry, and adapters. Authored rules for +choosing contexts, resolving conflicts, or promoting knowledge belong in +Vontology, VWL, or another independently governed semantic surface when a +capability actually needs them. + +Do not derive authority from context inclusion. Resolve the actor's maximum +visibility independently, then compute meaning only over authorised contexts. +Counts, lineage, conflicts, and error messages must not disclose inaccessible +contexts. + +Select only the claims material to the capability. Useful tests may show that: + +- base and actor-relative assertions remain distinguishable; +- provenance and source-context lineage survive an adapter; +- direct and inherited assertions are labelled when inheritance exists; +- conflict is not silently flattened; +- promotion creates a provenance-bearing assertion in the target context; or +- inclusion never enlarges visibility. + +Storage and index tests may still name collections where they protect an +adapter, migration, or Atlas query contract. They must not be the only evidence +for the knowledge behaviour. + +Add one neighbouring context case only when the claimed failure class is meant +to generalise. Require scale or query-plan evidence only for a scale claim. + +In Jira, name the generic failure as well as the domain symptom when that +distinction will help future work. A bounded issue may be Done when its user +outcome works; “Done” must not imply that Von's general theory architecture has +been selected. Link related work when useful, but do not make an architecture +programme or follow-up issue a prerequisite for a bounded repair. + +## 6. When more general machinery earns its cost + +Consider a first-class context substrate when a current capability needs it or +evidence shows recurring: + +- incompatible overlays or assertion stores; +- inability to explain direct versus inherited knowledge; +- real user, project, paper, hypothesis, or time-relative conflicts; +- migrations caused by conflating context, authority, provenance, and + publication; or +- query, authoring, or evaluation failures that a flat model cannot adequately + repair. + +Compare the proposed mechanism with the current flat or adapter baseline on the +complete user job: answer quality, lineage, conflict behaviour, non-leakage, +latency, Atlas cost, storage growth, migration cost, and authoring burden. + +## 7. Explicit exclusions + +This guide does not require: + +- universal context identifiers, automatic theory creation, a universal + assertion envelope, or a reasoner; +- a mandated store, migration, or logical formalism; +- an automatic Jira, review, build, or test gate; or +- delay to a useful bounded repair while a general architecture is unsettled. + +## 8. Related guidance + +- [Maintaining global design constraints and authority alignment](maintaining_global_design_constraints_and_authority_alignment_with_coding_agents.md) + covers the broader “one more local case” and test-pinning failure patterns. +- [Security considerations](security_considerations.md) governs actor, + namespace, and private-data authority. +- [Agent memory and enduring knowledge](agent_memory_and_enduring_knowledge.md) + governs promotion into durable memory. +- [Testing workflows and ephemeral theories](testing_workflows_ephemeral_theories_design.md) + and [Vontology tooling from KA/KR literature](vontology_tooling_from_ka_kcap_kr_literature.md) + are research and design inputs, not current implementation authority. diff --git a/src/backend/db/mongo_client.py b/src/backend/db/mongo_client.py index 9f26e7abb..5775a7a42 100644 --- a/src/backend/db/mongo_client.py +++ b/src/backend/db/mongo_client.py @@ -1140,6 +1140,27 @@ def _ensure_text_relations_indexes(coll: Collection) -> None: coll.create_index([("subject_concept_id", ASCENDING)], name="subject_concept_id_1") coll.create_index([("predicate", ASCENDING)], name="predicate_1") coll.create_index([("object_text_id", ASCENDING)], name="object_text_id_1") + coll.create_index( + [ + ("subject_concept_id", ASCENDING), + ("_id", ASCENDING), + ], + name="subject_concept_id_id", + ) + coll.create_index( + [ + ("predicate", ASCENDING), + ("_id", ASCENDING), + ], + name="predicate_id", + ) + coll.create_index( + [ + ("updated_at", ASCENDING), + ("_id", ASCENDING), + ], + name="updated_at_id", + ) coll.create_index( [ ("object_text_id", ASCENDING), @@ -1197,29 +1218,70 @@ def _ensure_scoped_knowledge_assertions_indexes(coll: Collection) -> None: name="assertion_id_unique", unique=True, ) + coll.create_index( + [ + ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), + ("updated_at", DESCENDING), + ("assertion_id", ASCENDING), + ], + name="audience_status_updated_at_desc_assertion_id", + ) coll.create_index( [ ("subject_concept_id", ASCENDING), ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), ("updated_at", DESCENDING), + ("assertion_id", ASCENDING), ], - name="subject_audience_updated_at_desc", + name="subject_audience_status_updated_at_desc_assertion_id", ) coll.create_index( [ ("object_concept_id", ASCENDING), ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), ("updated_at", DESCENDING), + ("assertion_id", ASCENDING), ], - name="object_audience_updated_at_desc", + name="object_audience_status_updated_at_desc_assertion_id", ) coll.create_index( [ ("predicate", ASCENDING), ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), ("updated_at", DESCENDING), + ("assertion_id", ASCENDING), + ], + name="predicate_audience_status_updated_at_desc_assertion_id", + ) + coll.create_index( + [ + ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), + ("_id", ASCENDING), + ], + name="audience_status_id", + ) + coll.create_index( + [ + ("subject_concept_id", ASCENDING), + ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), + ("_id", ASCENDING), + ], + name="subject_audience_status_id", + ) + coll.create_index( + [ + ("predicate", ASCENDING), + ("scope.audience_keys", ASCENDING), + ("status", ASCENDING), + ("_id", ASCENDING), ], - name="predicate_audience_updated_at_desc", + name="predicate_audience_status_id", ) diff --git a/src/backend/integrations/internal_mcp/catalogue.py b/src/backend/integrations/internal_mcp/catalogue.py index cbea1c02b..c61087bcc 100644 --- a/src/backend/integrations/internal_mcp/catalogue.py +++ b/src/backend/integrations/internal_mcp/catalogue.py @@ -87,6 +87,292 @@ def _run_in_worker(): return executor.submit(caller_context.run, _run_in_worker).result() +def _scoped_assertion_actor_claim( + kwargs: Mapping[str, Any], + field_names: Sequence[str], + *, + mismatch_field: str, +) -> Any: + """Return one compatible actor claim across legacy payload aliases.""" + + claimed_values: list[Any] = [] + normalised_values: set[str] = set() + for field_name in field_names: + value = kwargs.get(field_name) + if not isinstance(value, str) or not value.strip(): + continue + cleaned = value.strip() + normalised = cleaned if cleaned.startswith("#") else f"#V#{cleaned}" + claimed_values.append(value) + normalised_values.add(normalised) + if len(normalised_values) > 1: + raise WorkflowActorScopeError( + "workflow_actor_scope_mismatch", + mismatch_fields=(mismatch_field,), + ) + return claimed_values[0] if claimed_values else None + + +def _scoped_assertion_actor_scope_error_response( + exc: Exception, +) -> dict[str, Any]: + reason = str(getattr(exc, "reason", "") or "workflow_actor_scope_mismatch") + if reason == "workflow_actor_authority_required": + reason = "authenticated_actor_context_required" + mismatch_fields = getattr(exc, "mismatch_fields", ()) + return make_error_response( + reason, + ( + "Scoped-assertion access requires trusted actor provenance, and " + "payload identity claims must match that actor." + ), + details={ + "mismatch_fields": [ + str(item) + for item in mismatch_fields + if item in {"user_id", "org_id", "namespace"} + ] + }, + ) + + +def _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs: Mapping[str, Any], + *, + surface: str, + require_actor: bool = False, + ignore_untrusted_payload_identity: bool = False, +) -> tuple[Any | None, dict[str, Any] | None]: + """Resolve actor scope without promoting generic tool-payload identity. + + InternalMCP's default gateway installs user/organisation payload fields as a + compatibility access context. That context is not authentication. Scoped + assertion paths therefore bind only an actor that pre-dated the invocation, + an explicitly trusted operator fallback, or a deliberate direct in-process + call with no identity claims. Generic knowledge reads may instead discard + untrusted claims and bind no actor, preserving their base-publication view. + """ + + from ...security.access_control import ( + get_effective_organisation_concept_id, + get_effective_user_concept_id, + ) + from ...services.workflow_actor_scope_service import ( + resolve_authoritative_workflow_actor_scope, + ) + from .gateway import ( + get_internal_mcp_actor_context_source, + get_internal_mcp_preexisting_actor_context, + ) + + try: + source = get_internal_mcp_actor_context_source() + preexisting_actor = get_internal_mcp_preexisting_actor_context() + payload_claims_are_trusted = ( + source == "trusted_operator_payload_fallback" + ) + discard_payload_claims = ( + ignore_untrusted_payload_identity + and not payload_claims_are_trusted + ) + if discard_payload_claims: + claimed_user = None + claimed_org = None + claimed_namespace = None + else: + claimed_user = _scoped_assertion_actor_claim( + kwargs, + ( + "acting_user_concept_id", + "user_concept_id", + "user_id", + "actor_user_id", + ), + mismatch_field="user_id", + ) + claimed_org = _scoped_assertion_actor_claim( + kwargs, + ( + "organisation_concept_id", + "org_concept_id", + "organisation_id", + "org_id", + ), + mismatch_field="org_id", + ) + claimed_namespace = kwargs.get("namespace") + has_identity_claim = any( + isinstance(value, str) and bool(value.strip()) + for value in (claimed_user, claimed_org, claimed_namespace) + ) + + ambient_kwargs: dict[str, Any] + allow_unscoped_claims = False + if preexisting_actor is not None: + ambient_kwargs = { + "ambient_user_id": preexisting_actor[0], + "ambient_org_id": preexisting_actor[1], + "ambient_context_supplied": True, + } + elif source == "trusted_operator_payload_fallback": + ambient_kwargs = { + "ambient_user_id": None, + "ambient_org_id": None, + "ambient_context_supplied": False, + } + allow_unscoped_claims = True + elif source is None: + ambient_user = get_effective_user_concept_id() + ambient_org = get_effective_organisation_concept_id() + if ambient_user or ambient_org: + ambient_kwargs = { + "ambient_user_id": ambient_user, + "ambient_org_id": ambient_org, + "ambient_context_supplied": True, + } + elif not has_identity_claim: + # Preserve actorless direct unit/startup reads. They see no + # scoped rows because no audience is bound. + ambient_kwargs = { + "ambient_user_id": None, + "ambient_org_id": None, + "ambient_context_supplied": False, + } + allow_unscoped_claims = True + else: + raise WorkflowActorScopeError( + "workflow_actor_authority_required" + ) + elif source == "tool_payload_fallback" and not has_identity_claim: + if require_actor: + raise WorkflowActorScopeError( + "workflow_actor_authority_required" + ) + # Actorless generic reads retain their established base-publication + # behaviour. With no audience keys, lower services cannot expose + # scoped rows. + ambient_kwargs = { + "ambient_user_id": None, + "ambient_org_id": None, + "ambient_context_supplied": False, + } + allow_unscoped_claims = True + else: + # In particular, ``tool_payload_fallback`` is the ordinary gateway + # path whose identity originated in these same untrusted fields. + raise WorkflowActorScopeError("workflow_actor_authority_required") + + actor_scope = resolve_authoritative_workflow_actor_scope( + claimed_user_id=claimed_user, + claimed_org_id=claimed_org, + claimed_namespace=claimed_namespace, + allow_unscoped_claims=allow_unscoped_claims, + **ambient_kwargs, + ) + if require_actor and not bool( + actor_scope.user_concept_id + or actor_scope.organisation_concept_id + ): + raise WorkflowActorScopeError("workflow_actor_authority_required") + return actor_scope, None + except WorkflowActorScopeError as exc: + logger.info( + "Denied %s scoped-assertion access: %s", + surface, + getattr(exc, "reason", type(exc).__name__), + ) + return None, _scoped_assertion_actor_scope_error_response(exc) + + +@contextmanager +def _bind_internal_mcp_scoped_assertion_actor(actor_scope: Any): + """Apply only the actor scope returned by the provenance guard.""" + + from ...security.access_control import ( + force_access_control_enforcement, + override_current_actor, + ) + + with ( + override_current_actor( + actor_scope.user_concept_id, + actor_scope.organisation_concept_id, + ), + force_access_control_enforcement(), + ): + yield + + +def _internal_mcp_knowledge_context_view(actor_scope: Any) -> str: + """Select the semantic read view from verified actor authority only.""" + + if actor_scope.user_concept_id or actor_scope.organisation_concept_id: + return "actor_effective" + return "base_publication" + + +def _annotate_bounded_relation_lookup( + payload: Mapping[str, Any], + *, + concept_id: str, + predicate_filter: Any, + relation_kind: Any, +) -> dict[str, Any]: + """Make a truncated actor overlay explicit and non-pageable as a whole.""" + + result = dict(payload) + if not bool(result.get("total_hits_is_lower_bound")): + return result + + paging = dict(result.get("paging") or {}) + paging.update( + { + "continuation_supported": False, + "next_offset": None, + "offset_semantics": "bounded_actor_overlay_snapshot", + } + ) + result["paging"] = paging + + scoped_arguments: dict[str, Any] = { + "argument_concept_id": concept_id, + "limit": 200, + "offset": 0, + } + if isinstance(predicate_filter, list) and predicate_filter: + scoped_arguments["predicates"] = predicate_filter + relation_kind_token = str(relation_kind or "").strip().lower() + if relation_kind_token == "text": + scoped_arguments["object_kind"] = "text" + elif relation_kind_token == "binary": + scoped_arguments["object_kind"] = "concept" + + result["continuation"] = { + "status": "bounded_overlay_incomplete", + "can_continue_with_offset": False, + "reason": ( + "The actor-effective overlay was capped before combined sorting " + "and paging, so a later offset would not be a complete continuation." + ), + "recovery_options": [ + { + "action": "narrow_and_restart", + "restart_offset": 0, + "suggested_filters": [ + "predicate_filter", + "relation_kind", + ], + }, + { + "tool": "list_scoped_assertions", + "arguments": scoped_arguments, + "scope": "scoped_overlay_only", + }, + ], + } + return result + + def _get_vontology_tree(**kwargs): from ...vontology.utils_vontology import get_vontology_tree @@ -106,41 +392,60 @@ def _get_concept_by_concept_id(**kwargs): if not concept_id: raise ValueError("concept_id is required") - access = describe_concept_access(concept_id) - if access.get("exists") is True and access.get("accessible") is False: - return make_error_response( - "access_denied", - f"Concept is not accessible: {concept_id}", - details=access, - suggestions=[ - "Use search_concepts under the current namespace to find accessible concepts", - "Choose an accessible concept before attempting reads or writes", - ], - ) + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="concept enrichment", + ignore_untrusted_payload_identity=True, + ) + if denial is not None: + return denial + context_view = _internal_mcp_knowledge_context_view(actor_scope) + + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + access = describe_concept_access(concept_id) + if access.get("exists") is True and access.get("accessible") is False: + return make_error_response( + "access_denied", + f"Concept is not accessible: {concept_id}", + details=access, + suggestions=[ + "Use search_concepts under the current namespace to find accessible concepts", + "Choose an accessible concept before attempting reads or writes", + ], + ) - concept = get_concept_by_concept_id(concept_id=concept_id) + concept = get_concept_by_concept_id(concept_id=concept_id) - # Use shared enrichment logic (migrate-on-read + fetch names from text relations) - if not concept: - return concept + # Use shared enrichment logic (migrate-on-read + fetch names from text relations) + if not concept: + return concept - concept = enrich_concept_with_text_relations(concept) + concept = enrich_concept_with_text_relations(concept) - from ...services.scoped_assertion_service import ( - list_visible_scoped_assertions, - ) + scoped_assertion_rows: list[dict[str, Any]] = [] + if context_view == "actor_effective": + from ...services.scoped_assertion_service import ( + list_visible_scoped_assertions, + ) - scoped_assertions = list_visible_scoped_assertions( - subject_concept_ids=[concept_id], - limit=100, - ) - concept["scoped_assertions"] = scoped_assertions - concept["scoped_assertion_count"] = len(scoped_assertions) + scoped_assertion_rows = list_visible_scoped_assertions( + subject_concept_ids=[concept_id], + limit=101, + ) + scoped_assertions_truncated = len(scoped_assertion_rows) > 100 + scoped_assertions = scoped_assertion_rows[:100] + concept["context_view"] = context_view + concept["scoped_assertions"] = scoped_assertions + concept["scoped_assertion_count"] = len(scoped_assertions) + concept["scoped_assertions_truncated"] = scoped_assertions_truncated + concept["scoped_assertion_count_is_lower_bound"] = ( + scoped_assertions_truncated + ) - # Detect vacuous typing (soft warning for agents to repair) - vacuous_warning = detect_vacuous_typing(concept) - if vacuous_warning: - concept["_vacuous_typing_warning"] = vacuous_warning + # Detect vacuous typing (soft warning for agents to repair) + vacuous_warning = detect_vacuous_typing(concept) + if vacuous_warning: + concept["_vacuous_typing_warning"] = vacuous_warning include_relations_arg1 = bool(kwargs.get("include_relations_arg1")) include_relations_any_arg = bool(kwargs.get("include_relations_any_arg")) @@ -156,19 +461,20 @@ def _get_concept_by_concept_id(**kwargs): if any( [include_relations_arg1, include_relations_any_arg, include_text_relations_arg1] ): - relations_payload = build_concept_relations_payload( - concept, - include_relations_arg1=include_relations_arg1, - include_relations_any_arg=include_relations_any_arg, - include_text_relations_arg1=include_text_relations_arg1, - predicate_filter=predicate_filter, - limit=limit, - offset=offset, - include_concept_preview=include_concept_preview, - include_uncertain=include_uncertain, - uncertainty_mode=uncertainty_mode, - uncertainty_statuses=uncertainty_statuses, - ) + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + relations_payload = build_concept_relations_payload( + concept, + include_relations_arg1=include_relations_arg1, + include_relations_any_arg=include_relations_any_arg, + include_text_relations_arg1=include_text_relations_arg1, + predicate_filter=predicate_filter, + limit=limit, + offset=offset, + include_concept_preview=include_concept_preview, + include_uncertain=include_uncertain, + uncertainty_mode=uncertainty_mode, + uncertainty_statuses=uncertainty_statuses, + ) concept["relations"] = relations_payload return concept @@ -181,20 +487,37 @@ def _find_relations_with_argument(**kwargs): if concept_id is None or not str(concept_id).strip(): raise ValueError("concept_id is required") - return find_relations_with_argument( + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="relation lookup", + ignore_untrusted_payload_identity=True, + ) + if denial is not None: + return denial + context_view = _internal_mcp_knowledge_context_view(actor_scope) + + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + payload = find_relations_with_argument( + concept_id=str(concept_id), + argument_index=kwargs.get("argument_index"), + predicate_filter=kwargs.get("predicate_filter"), + relation_kind=kwargs.get("relation_kind"), + scope=kwargs.get("scope"), + include_text_snippets=bool(kwargs.get("include_text_snippets", False)), + include_concept_preview=bool(kwargs.get("include_concept_preview", True)), + limit=kwargs.get("limit"), + offset=kwargs.get("offset"), + sort_by=kwargs.get("sort_by"), + include_uncertain=bool(kwargs.get("include_uncertain", False)), + uncertainty_mode=kwargs.get("uncertainty_mode"), + uncertainty_statuses=kwargs.get("uncertainty_statuses"), + context_view=context_view, + ) + return _annotate_bounded_relation_lookup( + payload, concept_id=str(concept_id), - argument_index=kwargs.get("argument_index"), predicate_filter=kwargs.get("predicate_filter"), relation_kind=kwargs.get("relation_kind"), - scope=kwargs.get("scope"), - include_text_snippets=bool(kwargs.get("include_text_snippets", False)), - include_concept_preview=bool(kwargs.get("include_concept_preview", True)), - limit=kwargs.get("limit"), - offset=kwargs.get("offset"), - sort_by=kwargs.get("sort_by"), - include_uncertain=bool(kwargs.get("include_uncertain", False)), - uncertainty_mode=kwargs.get("uncertainty_mode"), - uncertainty_statuses=kwargs.get("uncertainty_statuses"), ) @@ -2087,15 +2410,75 @@ def _upsert_scoped_assertion(**kwargs): "Missing 'predicate' parameter", details={"missing": ["predicate"]}, ) + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="scoped assertion mutation", + require_actor=True, + ) + if denial is not None: + return denial + if actor_scope.user_concept_id is None: + return make_error_response( + "authenticated_actor_context_required", + "Scoped assertion mutation requires an authenticated user actor.", + ) + + service_kwargs = { + field_name: kwargs.get(field_name) + for field_name in ( + "subject_concept_id", + "predicate", + "target_text", + "target_concept_id", + "language", + "scope_mode", + "evidence", + "turn_id", + ) + if field_name in kwargs + } + service_kwargs.update( + { + "acting_user_concept_id": actor_scope.user_concept_id, + "organisation_concept_id": actor_scope.organisation_concept_id, + "namespace": actor_scope.namespace, + "canonical_publication": False, + } + ) try: - result = upsert_scoped_assertion(**kwargs) + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + result = upsert_scoped_assertion(**service_kwargs) assertion = result.get("assertion") if isinstance(assertion, dict) and assertion.get("object_kind") == "text": - maybe_sync_concept_text_relations_to_rag( - namespace=kwargs.get("namespace"), - concept_id=str(subject_concept_id), - predicate=str(assertion.get("predicate") or predicate), - ) + try: + derived_maintenance = maybe_sync_concept_text_relations_to_rag( + namespace=actor_scope.namespace, + concept_id=str(subject_concept_id), + predicate=str(assertion.get("predicate") or predicate), + scoped_assertion_id=assertion.get("assertion_id"), + ) + except Exception as exc: + # The canonical assertion and its read-back already succeeded. + # Derived-index maintenance must not turn that durable receipt + # into an indeterminate write result. + logger.warning( + "Scoped assertion persisted but RAG refresh scheduling " + "failed: %s", + exc, + ) + derived_maintenance = { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "schedule_failed", + "error": str(exc), + } + result = { + **result, + "derived_maintenance": derived_maintenance, + } return result except PermissionError as exc: return make_error_response( @@ -2120,9 +2503,102 @@ def _upsert_scoped_assertion(**kwargs): ) +def _retract_scoped_assertion(**kwargs): + from ...services.rag_text_relation_change_hook_service import ( + maybe_sync_concept_text_relations_to_rag, + ) + from ...services.scoped_assertion_service import retract_scoped_assertion + + assertion_id = kwargs.get("assertion_id") + if not assertion_id: + return make_error_response( + "missing_parameter", + "Missing 'assertion_id' parameter", + details={"missing": ["assertion_id"]}, + ) + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="scoped assertion retraction", + require_actor=True, + ) + if denial is not None: + return denial + if actor_scope.user_concept_id is None: + return make_error_response( + "authenticated_actor_context_required", + "Scoped assertion retraction requires an authenticated user actor.", + ) + + try: + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + result = retract_scoped_assertion( + assertion_id=assertion_id, + acting_user_concept_id=actor_scope.user_concept_id, + organisation_concept_id=actor_scope.organisation_concept_id, + namespace=actor_scope.namespace, + ) + assertion = result.get("assertion") + if ( + isinstance(assertion, dict) + and assertion.get("object_kind") == "text" + and assertion.get("subject_concept_id") + and assertion.get("predicate") + ): + try: + derived_maintenance = maybe_sync_concept_text_relations_to_rag( + namespace=actor_scope.namespace, + concept_id=str(assertion["subject_concept_id"]), + predicate=str(assertion["predicate"]), + scoped_assertion_id=str( + assertion.get("assertion_id") or assertion_id + ), + ) + except Exception as exc: + logger.warning( + "Scoped assertion retracted but RAG refresh scheduling " + "failed: %s", + exc, + ) + derived_maintenance = { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "schedule_failed", + "error": str(exc), + } + result = { + **result, + "derived_maintenance": derived_maintenance, + } + return result + except PermissionError as exc: + return make_error_response( + "access_denied", + str(exc), + details={"assertion_id": str(assertion_id)}, + ) + except ValueError as exc: + return make_error_response( + "invalid_scoped_assertion", + str(exc), + details={"assertion_id": str(assertion_id)}, + ) + except Exception as exc: + return _indeterminate_effect_error( + "The scoped-assertion retraction raised unexpectedly.", + details={ + "exception_type": type(exc).__name__, + "exception": str(exc), + "assertion_id": str(assertion_id), + }, + ) + + def _list_scoped_assertions(**kwargs): from ...services.scoped_assertion_service import ( - list_visible_scoped_assertions, + list_visible_scoped_assertions_page, ) subject_concept_id = kwargs.get("subject_concept_id") @@ -2130,19 +2606,47 @@ def _list_scoped_assertions(**kwargs): predicate = kwargs.get("predicate") if predicate and not predicates: predicates = [predicate] - assertions = list_visible_scoped_assertions( - subject_concept_ids=([str(subject_concept_id)] if subject_concept_id else None), - argument_concept_id=kwargs.get("argument_concept_id"), - predicates=predicates, - object_kind=kwargs.get("object_kind"), - limit=kwargs.get("limit") or 200, - user_concept_id=kwargs.get("acting_user_concept_id"), - organisation_concept_id=kwargs.get("organisation_concept_id"), + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="scoped assertion listing", + require_actor=True, ) + if denial is not None: + return denial + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + page = list_visible_scoped_assertions_page( + subject_concept_ids=( + [str(subject_concept_id)] if subject_concept_id else None + ), + argument_concept_id=kwargs.get("argument_concept_id"), + predicates=predicates, + object_kind=kwargs.get("object_kind"), + languages=kwargs.get("languages"), + limit=kwargs.get("limit") or 200, + offset=kwargs.get("offset") or 0, + user_concept_id=actor_scope.user_concept_id, + organisation_concept_id=actor_scope.organisation_concept_id, + ) + assertions = list(page.get("items") or ()) + counts_are_lower_bounds = bool(page.get("counts_are_lower_bounds")) return { "success": True, "assertions": assertions, "assertions_found": len(assertions), + "paging": { + "limit": page.get("limit"), + "offset": page.get("offset"), + "returned": page.get("returned"), + "has_more": bool(page.get("has_more")), + "next_offset": page.get("next_offset"), + "visibility_filtered": bool(page.get("visibility_filtered")), + "counts_are_lower_bounds": counts_are_lower_bounds, + }, + **( + {"assertions_found_is_lower_bound": True} + if counts_are_lower_bounds + else {} + ), "canonical_publication": False, } @@ -2166,19 +2670,30 @@ def _get_text_relations(**kwargs): suggestions=["Provide the concept ID to retrieve text relations for"], ) + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="text retrieval", + ignore_untrusted_payload_identity=True, + ) + if denial is not None: + return denial + context_view = _internal_mcp_knowledge_context_view(actor_scope) + try: - relations = get_texts_for_concept( - subject_concept_id=concept_id, - predicate=predicate, - lang=language, - limit=limit, - recent_first=( - recent_first - if isinstance(recent_first, bool) - else str(recent_first or "").strip().lower() - in {"1", "true", "yes", "on"} - ), - ) + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + relations = get_texts_for_concept( + subject_concept_id=concept_id, + predicate=predicate, + lang=language, + limit=limit, + context_view=context_view, + recent_first=( + recent_first + if isinstance(recent_first, bool) + else str(recent_first or "").strip().lower() + in {"1", "true", "yes", "on"} + ), + ) # Add text previews for long content for relation in relations: @@ -2188,6 +2703,7 @@ def _get_text_relations(**kwargs): return { "concept_id": concept_id, + "context_view": context_view, "relations_found": len(relations), "relations": relations, } @@ -2371,13 +2887,24 @@ def _get_text_relations_summary(**kwargs): suggestions=["Provide the concept ID to get text relations summary for"], ) + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="text summary", + ignore_untrusted_payload_identity=True, + ) + if denial is not None: + return denial + context_view = _internal_mcp_knowledge_context_view(actor_scope) + try: - return get_text_relations_summary( - concept_id, - predicates=predicates, - languages=languages, - max_relation_ids_per_group=max_relation_ids_per_group, - ) + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + return get_text_relations_summary( + concept_id, + predicates=predicates, + languages=languages, + max_relation_ids_per_group=max_relation_ids_per_group, + context_view=context_view, + ) except Exception as exc: return make_error_response( "exception", @@ -8792,12 +9319,48 @@ def _upsert_scoped_assertion_output_schema() -> Schema: "canonical_read_back": (dict, type(None)), "canonical_publication": (bool, type(None)), "storage_surface": (str, type(None)), + "derived_maintenance": (dict, type(None)), "error": (str, type(None)), }, allow_unknown=True, description=( "Scoped assertion write receipt with durable read-back. " - "canonical_publication is always false." + "canonical_publication is always false. derived_maintenance reports " + "best-effort index refresh scheduling without changing effect success." + ), + ) + + +def _retract_scoped_assertion_input_schema() -> Schema: + return Schema( + required={"assertion_id": str}, + optional={}, + allow_unknown=True, + description=( + "retract_scoped_assertion input: exact assertion_id only. Actor, " + "organisation, and namespace are server-bound." + ), + ) + + +def _retract_scoped_assertion_output_schema() -> Schema: + return Schema( + required={"success": bool}, + optional={ + "effect_status": (str, type(None)), + "changed": (bool, type(None)), + "assertion_id": (str, type(None)), + "assertion": (dict, type(None)), + "canonical_read_back": (dict, type(None)), + "canonical_publication": (bool, type(None)), + "storage_surface": (str, type(None)), + "derived_maintenance": (dict, type(None)), + "error": (str, type(None)), + }, + allow_unknown=True, + description=( + "Idempotent scoped-assertion retraction receipt with durable " + "read-back. derived_maintenance is best-effort and non-authoritative." ), ) @@ -8811,12 +9374,15 @@ def _list_scoped_assertions_input_schema() -> Schema: "predicate": (str, type(None)), "predicates": (list, type(None)), "object_kind": (str, type(None)), + "languages": (list, type(None)), "limit": (int, type(None)), + "offset": (int, type(None)), }, allow_unknown=True, description=( "List assertions visible in the trusted user/organisation scope, " - "optionally filtered by subject, argument, predicate, or object kind." + "bounded by limit/offset and optionally filtered by subject, argument, " + "predicate, object kind, or text language." ), ) @@ -8829,8 +9395,15 @@ def _list_scoped_assertions_output_schema() -> Schema: "assertions_found": int, "canonical_publication": bool, }, - optional={}, + optional={ + "assertions_found_is_lower_bound": (bool, type(None)), + "paging": (dict, type(None)), + }, allow_unknown=True, + description=( + "Visible actor-scoped assertions with paging and completeness metadata; " + "these rows are not canonical publication." + ), ) @@ -8857,10 +9430,16 @@ def _get_text_relations_output_schema() -> Schema: }, optional={ "relations": (list, type(None)), + "context_view": (str, type(None)), "error": (str, type(None)), }, allow_unknown=True, - description="get_text_relations output: concept_id (str), relations_found (int), relations (list of {text, lang, text_value_id, predicate, relation_id, context, text_preview?}), error (str if failed)", + description=( + "get_text_relations output: context_view labels base_publication or " + "actor_effective rows. Inspect row_kind. base_text_relation rows have " + "a relation_id usable by update/delete; scoped_assertion rows have " + "assertion_id and relation_id=null." + ), ) @@ -8875,7 +9454,11 @@ def _update_text_relation_input_schema() -> Schema: "language": (str, type(None)), }, allow_unknown=True, - description="update_text_relation input: concept_id (str), relation_id (str), new_text (str), language (str, optional)", + description=( + "update_text_relation input: concept_id, relation_id, new_text, and " + "optional language. relation_id must come from a base_text_relation " + "row; scoped_assertion IDs are not mutable text-relation IDs." + ), ) @@ -8909,7 +9492,12 @@ def _delete_text_relation_input_schema() -> Schema: "garbage_collect": (bool,), }, allow_unknown=True, - description="delete_text_relation input: concept_id (str), relation_id (str, optional - preferred method), predicate (str, optional for predicate+text deletion), text (str, optional with predicate), language (str, optional), garbage_collect (bool, optional - if true, delete orphaned text_values)", + description=( + "delete_text_relation input: concept_id plus a base_text_relation " + "relation_id (preferred), or predicate+text and optional language. " + "Scoped assertion IDs are not text-relation IDs. garbage_collect " + "optionally deletes orphaned text_values." + ), ) @@ -8936,9 +9524,19 @@ def _get_text_relations_summary_output_schema() -> Schema: "total_relations_scanned": int, "max_relation_ids_per_group": int, }, - optional={"error": (str, type(None))}, + optional={ + "context_view": (str, type(None)), + "scoped_query_truncated": (bool, type(None)), + "counts_are_lower_bounds": (bool, type(None)), + "error": (str, type(None)), + }, allow_unknown=True, - description="get_text_relations_summary output: success, concept_id, groups[{predicate, language, count, relation_ids, latest_relation_id, latest_updated_at}], groups_found, total_relations_scanned", + description=( + "get_text_relations_summary output: context-labelled grouped counts. " + "relation_ids and assertion_ids remain separate; row-specific latest " + "IDs are labelled. In actor_effective view, scoped_query_truncated and " + "counts_are_lower_bounds make the bounded overlay explicit." + ), ) @@ -9147,13 +9745,19 @@ def _find_relations_with_argument_output_schema() -> Schema: "hits": list, "paging": dict, }, - optional={}, + optional={ + "context_view": (str, type(None)), + "total_hits_is_lower_bound": (bool, type(None)), + "continuation": (dict, type(None)), + }, allow_unknown=True, description=( "find_relations_with_argument output: concept_id, total_hits, hits[] " "(source_concept_id, predicate_concept_id, relation_kind, argument_indexes, " "target_value, optional previews/snippets, relation_metadata, access_granted, " - "follow_up_actions, score, optional uncertainty metadata), and paging metadata." + "follow_up_actions, score, optional uncertainty metadata), context_view, " + "and paging metadata. A truncated actor overlay is labelled as a lower " + "bound and disables offset continuation, with recovery options." ), ) @@ -11242,9 +11846,13 @@ def _finalise_report(report: dict[str, Any]) -> dict[str, Any]: ) return enriched - from .gateway import get_internal_mcp_preexisting_actor_context + from .gateway import ( + get_internal_mcp_actor_context_source, + get_internal_mcp_preexisting_actor_context, + ) preexisting_actor = get_internal_mcp_preexisting_actor_context() + actor_context_source = get_internal_mcp_actor_context_source() if preexisting_actor is not None: trusted_user = _normalise_concept_id(preexisting_actor[0]) trusted_org = _normalise_concept_id(preexisting_actor[1]) @@ -11289,6 +11897,23 @@ def _finalise_report(report: dict[str, Any]) -> dict[str, Any]: } ) + if actor_context_source == "tool_payload_fallback": + # The default InternalMCP gateway installs payload identity only as a + # compatibility context. It is not authentication and must never select + # another actor's RAG namespace or permissions. + return _finalise_report( + { + "namespace": None, + "namespace_source": "untrusted_payload_rejected", + "namespace_resolution_note": ( + "authenticated_actor_context_required" + ), + "namespace_mismatch": False, + "provided_namespace": explicit_namespace, + "derived_namespace": None, + } + ) + if ( explicit_namespace and derived_namespace @@ -17276,61 +17901,86 @@ def _search_knowledge_base(**kwargs): start = time.perf_counter() try: - service = get_rag_service() # Default backend + # The namespace resolver is the authority boundary for actor-private + # RAG access. Build row-level permissions from its accepted trusted + # components, never from raw tool arguments. + permissions_context: dict[str, Any] = {} + resolved_user_concept_id = ns_report.get("derived_user_concept_id") + if ( + isinstance(resolved_user_concept_id, str) + and resolved_user_concept_id.strip() + ): + permissions_context["user_id"] = resolved_user_concept_id.strip() + resolved_org_concept_id = ns_report.get( + "derived_organisation_concept_id" + ) + if ( + isinstance(resolved_org_concept_id, str) + and resolved_org_concept_id.strip() + ): + permissions_context["organisation_concept_id"] = ( + resolved_org_concept_id.strip() + ) - # Build permissions context from Flask session for org-scoped RAG filtering. - # IMPORTANT: Use concept IDs (e.g. #V#user) rather than email/usernames. - permissions_context = {} + # Flask identity is a trusted compatibility source, but may only fill + # a component the accepted namespace report did not establish. A + # conflicting session must fail closed rather than selecting one + # actor's physical namespace with another actor's row permissions. try: from flask import session as flask_session - user_concept_id = flask_session.get("user_concept_id") - if isinstance(user_concept_id, str) and user_concept_id.strip(): - permissions_context["user_id"] = user_concept_id.strip() - elif flask_session.get("user_id"): + session_user_concept_id = flask_session.get("user_concept_id") + if not ( + isinstance(session_user_concept_id, str) + and session_user_concept_id.strip() + ) and flask_session.get("user_id"): # Backwards compatibility: some sessions store a non-concept user_id. - # Fall back to that only if we don't have a concept ID. - permissions_context["user_id"] = flask_session.get("user_id") - org_concept_id = flask_session.get("organisation_concept_id") - if not org_concept_id: + session_user_concept_id = flask_session.get("user_id") + session_org_concept_id = flask_session.get( + "organisation_concept_id" + ) + if not session_org_concept_id: # Backwards compatibility for older session key. - org_concept_id = flask_session.get("org_id") - if org_concept_id: - permissions_context["organisation_concept_id"] = org_concept_id + session_org_concept_id = flask_session.get("org_id") except (ImportError, RuntimeError): - # Not in Flask context (e.g., external MCP stdio server). - # Derive the same effective filtering keys we use in-server when possible. - user = kwargs.get("user") - user_id = user.get("id") if isinstance(user, dict) else None - if isinstance(user_id, str) and user_id.strip(): - # IMPORTANT: Use concept IDs verbatim (case sensitive, e.g. #V#person). - permissions_context["user_id"] = user_id.strip() - - org_concept_id = kwargs.get("organisation_concept_id") - if isinstance(org_concept_id, str) and org_concept_id.strip(): - permissions_context["organisation_concept_id"] = org_concept_id.strip() - elif ( - isinstance(kwargs.get("org_id"), str) - and str(kwargs.get("org_id")).strip() - ): - permissions_context["organisation_concept_id"] = str( - kwargs.get("org_id") - ).strip() + session_user_concept_id = None + session_org_concept_id = None - # If the namespace is in the user@org form, it contains enough - # information to derive both IDs without trusting arbitrary inputs. - if isinstance(ns, str) and "@" in ns: - user_part, org_part = ns.split("@", 1) - if user_part.strip() and "user_id" not in permissions_context: - permissions_context["user_id"] = user_part.strip() - if ( - org_part.strip() - and "organisation_concept_id" not in permissions_context - ): - org_part_clean = org_part.strip() - if not org_part_clean.startswith("#V#"): - org_part_clean = f"#V#{org_part_clean}" - permissions_context["organisation_concept_id"] = org_part_clean + def _clean_identity_component(value: Any) -> str | None: + if not isinstance(value, str) or not value.strip(): + return None + cleaned = value.strip() + if cleaned.startswith("#v#"): + return f"#V#{cleaned[3:]}" + return cleaned if cleaned.startswith("#V#") else f"#V#{cleaned}" + + for permission_key, session_value in ( + ("user_id", session_user_concept_id), + ("organisation_concept_id", session_org_concept_id), + ): + session_component = _clean_identity_component(session_value) + if session_component is None: + continue + resolved_component = _clean_identity_component( + permissions_context.get(permission_key) + ) + if ( + resolved_component is not None + and resolved_component != session_component + ): + return make_error_response( + "namespace_mismatch", + ( + "Authenticated session identity conflicts with the " + "resolved RAG namespace; request was not executed." + ), + details={ + "mismatch_fields": [permission_key], + "namespace_source": ns_report.get("namespace_source"), + }, + ) + if resolved_component is None: + permissions_context[permission_key] = session_component # Optional semantic filtering (handled by the backend). mode = kwargs.get("mode") @@ -17349,6 +17999,7 @@ def _search_knowledge_base(**kwargs): if isinstance(predicates, list): permissions_context["predicates"] = predicates + service = get_rag_service() # Default backend results = service.query( query_text=query_text, top_k=kwargs.get("top_k", 5), @@ -22529,13 +23180,6 @@ def _build_rag_file_copy_item( def _rag_list_indexed(**kwargs): from ...db.connection_manager import get_db - db = get_db() - if db is None: - return make_error_response( - "db_unavailable", - "Database connection unavailable", - suggestions=["Check database connectivity and configuration"], - ) collection_report = _resolve_rag_collection_from_kwargs(kwargs) collection = collection_report.get("effective_collection") limit = max(1, _normalise_non_negative_int(kwargs.get("limit") or 20)) @@ -22556,6 +23200,14 @@ def _rag_list_indexed(**kwargs): ) ns = ns.strip() + db = get_db() + if db is None: + return make_error_response( + "db_unavailable", + "Database connection unavailable", + suggestions=["Check database connectivity and configuration"], + ) + if not isinstance(collection, str) or not collection: collection = "ka_sessions" @@ -23518,11 +24170,23 @@ def _safe_bool_optional(value: Any) -> bool | None: for row in raw_items: if not isinstance(row, dict): continue + row_kind = ( + str(row.get("row_kind") or "base_text_relation").strip() + or "base_text_relation" + ) + relation_id = row.get("relation_id") + assertion_id = row.get("assertion_id") + index_item_id = row.get("index_item_id") items.append( { "collection": collection, - "session_id": row.get("relation_id"), - "relation_id": row.get("relation_id"), + "session_id": ( + relation_id or assertion_id or index_item_id + ), + "index_item_id": index_item_id, + "row_kind": row_kind, + "relation_id": relation_id, + "assertion_id": assertion_id, "subject_concept_id": row.get("subject_concept_id"), "predicate": row.get("predicate"), "lang": row.get("lang"), @@ -23530,7 +24194,11 @@ def _safe_bool_optional(value: Any) -> bool | None: "updated_at": row.get("updated_at"), "namespace": ns, "item_kind": "vontology_text_relation", - "source_system": "mongo.text_relations", + "source_system": ( + "mongo.scoped_knowledge_assertions" + if row_kind == "scoped_assertion" + else "mongo.text_relations" + ), "namespace_source": ns_report.get("namespace_source"), } ) @@ -23574,13 +24242,6 @@ def _rag_get_item(**kwargs): from ...db.connection_manager import get_db from bson import ObjectId - db = get_db() - if db is None: - return make_error_response( - "db_unavailable", - "Database connection unavailable", - suggestions=["Check database connectivity and configuration"], - ) collection_report = _resolve_rag_collection_from_kwargs(kwargs) collection = collection_report.get("effective_collection") session_id = kwargs.get("session_id") @@ -23607,6 +24268,14 @@ def _rag_get_item(**kwargs): ) ns = ns.strip() + db = get_db() + if db is None: + return make_error_response( + "db_unavailable", + "Database connection unavailable", + suggestions=["Check database connectivity and configuration"], + ) + if not isinstance(collection, str) or not collection: collection = "ka_sessions" @@ -24139,14 +24808,23 @@ def _rag_get_item(**kwargs): "collection": collection, **collection_report, "session_id": session_id, + "index_item_id": doc.doc_id, + "row_kind": ( + doc.metadata.get("row_kind") or "base_text_relation" + ), "relation_id": doc.metadata.get("relation_id"), + "assertion_id": doc.metadata.get("assertion_id"), "subject_concept_id": doc.metadata.get("subject_concept_id"), "predicate": doc.metadata.get("predicate"), "lang": doc.metadata.get("lang"), "namespace": ns, "preview": (doc.text or "")[:4000], "item_kind": "vontology_text_relation", - "source_system": "mongo.text_relations", + "source_system": ( + "mongo.scoped_knowledge_assertions" + if doc.metadata.get("row_kind") == "scoped_assertion" + else "mongo.text_relations" + ), "namespace_source": ns_report.get("namespace_source"), "effective_namespace": ns, "effective_namespace_source": ns_report.get("namespace_source"), @@ -32993,7 +33671,12 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: input_schema=_concept_fetch_input_schema(), output_schema=None, category="read", - description="Fetch full details of ONE specific concept by its ID (format: #V#concept_name). Use when you already know the exact concept_id and need complete information (description, predicates, relationships). Don't use for searching.", + description=( + "Fetch full details of one concept by exact ID. An authenticated " + "actor receives actor_effective scoped enrichment; identity or " + "namespace supplied only in an ordinary tool payload is ignored and " + "the response is explicitly base_publication. Don't use for searching." + ), ), MethodDefinition( name="find_relations_with_argument", @@ -33007,7 +33690,10 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: "argument_index to choose the relation slot instead of inventing " "subject/object payload fields. Supports exact graph matching and " "full-text text-relation matching with optional predicate/kind filters " - "and pagination." + "and pagination. Verified actors receive actor_effective results; " + "payload-only identity is ignored and yields base_publication. " + "Truncated actor overlays expose bounded recovery rather than a " + "misleading offset continuation." ), ), MethodDefinition( @@ -33193,6 +33879,27 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: "knowledge intended to be shared with the authenticated organisation." ), ), + MethodDefinition( + name="retract_scoped_assertion", + handler=_retract_scoped_assertion, + input_schema=_retract_scoped_assertion_input_schema(), + output_schema=_retract_scoped_assertion_output_schema(), + category="write", + ordinary_turn_trusted_argument_bindings={ + "acting_user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + "namespace": "turn_namespace", + }, + ordinary_turn_effect=True, + effect_admission_window_sec=8.0, + description=( + "Idempotently retract one actor-scoped assertion by exact " + "assertion_id. The server binds actor, organisation, and namespace; " + "only the original authorised author can retract it. The durable " + "receipt remains successful even if derived-index refresh scheduling " + "fails, with that maintenance result reported separately." + ), + ), MethodDefinition( name="list_scoped_assertions", handler=_list_scoped_assertions, @@ -33205,7 +33912,8 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: }, description=( "Read provenance-bearing non-canonical assertions visible to the " - "trusted user or organisation, with bounded filters." + "trusted user or organisation, with bounded filters and paging. This " + "dedicated scoped read requires trusted actor provenance." ), ), MethodDefinition( @@ -33223,10 +33931,11 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: ordinary_turn_mutation_subject_argument="concept_id", description=( "Add or update a canonical text relation when the actor has mutation " - "authority over the subject concept. For knowledge about a visible " - "concept the actor does not own, use upsert_scoped_assertion instead. " - "Supports hasContent, hasDescription, hasNote, hasName, and existing " - "custom predicate concepts." + "authority over the subject concept. On an actor-bound surface that " + "exposes upsert_scoped_assertion, use that capability for knowledge " + "about a visible concept the actor does not own. Supports hasContent, " + "hasDescription, hasNote, hasName, and existing custom predicate " + "concepts." ), ), MethodDefinition( @@ -33235,7 +33944,14 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: input_schema=_get_text_relations_input_schema(), output_schema=_get_text_relations_output_schema(), category="read", - description="Retrieve text relations for a concept, optionally filtered by predicate/language. Returns all text attachments (hasContent, hasDescription, hasName, etc.). Use to query what text is attached to a concept.", + description=( + "Retrieve labelled text-assertion rows for a concept, optionally " + "filtered by predicate/language. The serving authority boundary " + "selects context_view: trusted actor-bound internal calls use " + "actor_effective; actorless calls and ordinary payload-only identity " + "use base_publication. Inspect row_kind; only a " + "base_text_relation relation_id is valid for update/delete." + ), ), MethodDefinition( name="update_text_relation", @@ -33259,7 +33975,14 @@ def _build_default_catalogue_core_definitions() -> List[MethodDefinition]: input_schema=_get_text_relations_summary_input_schema(), output_schema=_get_text_relations_summary_output_schema(), category="read", - description="Return a lightweight summary of text relations for a concept: counts + relation IDs grouped by predicate/language (no full text bodies). Use to quickly decide what to fetch next.", + description=( + "Return a lightweight, context-labelled text-assertion summary " + "grouped by predicate/language (no full text bodies). The serving " + "authority boundary selects actor_effective or base_publication; " + "ordinary payload-only identity is ignored. " + "Relation and assertion IDs remain separate; inspect " + "counts_are_lower_bounds before treating scoped counts as complete." + ), ), MethodDefinition( name="upsert_singleton_text_relation", @@ -34234,6 +34957,13 @@ def _build_default_catalogue_knowledge_io_definitions() -> List[MethodDefinition output_schema=_search_knowledge_base_output_schema(), category="read", timeout_sec=30.0, + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + "user_concept_id": "actor_user_concept_id", + "organisation_concept_id": ( + "actor_organisation_concept_id" + ), + }, description="Search the internal knowledge base (RAG) for documents and indexed content. Use when user asks about internal documents, policies, or specific indexed knowledge that is not in the ontology or on the public web. Returns semantically relevant text chunks.", ), MethodDefinition( @@ -34243,6 +34973,13 @@ def _build_default_catalogue_knowledge_io_definitions() -> List[MethodDefinition output_schema=_search_concept_descriptions_output_schema(), category="read", timeout_sec=30.0, + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + "user_concept_id": "actor_user_concept_id", + "organisation_concept_id": ( + "actor_organisation_concept_id" + ), + }, description=( "Semantic search over concept descriptions only (hasDescription text relations) for the current namespace. " "Use when user asks to search the knowledge base but only within concept descriptions." @@ -34255,6 +34992,13 @@ def _build_default_catalogue_knowledge_io_definitions() -> List[MethodDefinition output_schema=_get_related_concepts_output_schema(), category="read", timeout_sec=30.0, + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + "user_concept_id": "actor_user_concept_id", + "organisation_concept_id": ( + "actor_organisation_concept_id" + ), + }, description=( "Find concepts with similar descriptions (vector similarity) within the current namespace. " "Returns description chunks and metadata for related concepts." @@ -34896,6 +35640,9 @@ def _build_default_catalogue_diagnostics_and_research_definitions() -> ( ), output_schema=None, category="read", + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + }, description=( "List available RAG collections/sources for the current user/namespace. " "Use when user asks 'what is in my RAG store?' or needs to disambiguate KA sessions, " @@ -34918,6 +35665,9 @@ def _build_default_catalogue_diagnostics_and_research_definitions() -> ( ), output_schema=None, category="read", + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + }, description="List namespace-scoped RAG items for the selected collection (KA sessions, chat sessions, file-copy concepts, turn execution records, experiment runs, text relations).", ), MethodDefinition( @@ -34934,6 +35684,9 @@ def _build_default_catalogue_diagnostics_and_research_definitions() -> ( ), output_schema=None, category="read", + ordinary_turn_trusted_argument_bindings={ + "namespace": "turn_namespace", + }, description="Get one namespace-scoped RAG item (session/relation/file copy/turn record) with a safe preview. Respects namespace isolation.", ), MethodDefinition( diff --git a/src/backend/mcp_server/mcp_stdio_server.py b/src/backend/mcp_server/mcp_stdio_server.py index 8552b2cd7..3434a51cc 100644 --- a/src/backend/mcp_server/mcp_stdio_server.py +++ b/src/backend/mcp_server/mcp_stdio_server.py @@ -201,7 +201,7 @@ get_search_proxy, ) from src.backend.integrations.google import gmail_service - from src.backend.services.rag_service import RAGBackendUnavailable, get_rag_service + from src.backend.services.rag_service import RAGBackendUnavailable # Avoid UnicodeEncodeError on Windows consoles (default cp1252) when any # dependency logs Unicode (e.g. checkmarks). MCP runs over stdio; we must not @@ -486,7 +486,7 @@ def _bind_imports(module_name: str, names: list[str]) -> None: gmail_service = importlib.import_module("src.backend.integrations.google.gmail_service") _bind_imports( "src.backend.services.rag_service", - ["get_rag_service", "RAGBackendUnavailable"], + ["RAGBackendUnavailable"], ) # Create MCP server instance @@ -1706,13 +1706,21 @@ async def _handle_get_text_relations(arguments: dict[str, Any]) -> list[TextCont ] try: - relations = get_texts_for_concept( - subject_concept_id=concept_id, - predicate=predicate, - lang=language, - limit=limit, + from src.backend.security.access_control import ( + force_access_control_enforcement, ) + with force_access_control_enforcement(): + relations = get_texts_for_concept( + subject_concept_id=concept_id, + predicate=predicate, + lang=language, + limit=limit, + # Direct stdio calls have no authenticated actor scope. Keep + # this surface on the globally visible base publication view. + context_view="base_publication", + ) + # Add text previews for long content for relation in relations: text = relation.get("text", "") @@ -1721,6 +1729,7 @@ async def _handle_get_text_relations(arguments: dict[str, Any]) -> list[TextCont payload = { "concept_id": concept_id, + "context_view": "base_publication", "relations_found": len(relations), "relations": relations, } @@ -2046,73 +2055,84 @@ async def _handle_fetch_concept(arguments: dict[str, Any]) -> list[TextContent]: ] try: - concept = get_concept_by_concept_id(concept_id) - if not concept: - return [ - _json_error( - f"Concept '{concept_id}' not found", - error_code="concept_not_found", - suggestions=[ - "Check the concept_id spelling", - "Use search_concepts to find available concepts", - ], - related_concept_ids=[concept_id], - ) - ] + from src.backend.security.access_control import ( + force_access_control_enforcement, + ) - concept = enrich_concept_with_text_relations(concept) + with force_access_control_enforcement(): + concept = get_concept_by_concept_id(concept_id) + if not concept: + return [ + _json_error( + f"Concept '{concept_id}' not found", + error_code="concept_not_found", + suggestions=[ + "Check the concept_id spelling", + "Use search_concepts to find available concepts", + ], + related_concept_ids=[concept_id], + ) + ] - # NOTE: mcp_stdio_server runs as a top-level script in stdio mode. - # Relative imports fail in that runtime ("no known parent package"), - # so this import must remain absolute. - from src.backend.services.relationship_write_service import ( - detect_vacuous_typing, - ) + concept = enrich_concept_with_text_relations(concept) - vacuous_warning = detect_vacuous_typing(concept) - if vacuous_warning: - concept["_vacuous_typing_warning"] = vacuous_warning + # NOTE: mcp_stdio_server runs as a top-level script in stdio mode. + # Relative imports fail in that runtime ("no known parent package"), + # so this import must remain absolute. + from src.backend.services.relationship_write_service import ( + detect_vacuous_typing, + ) - include_relations_arg1 = bool(arguments.get("include_relations_arg1")) - include_relations_any_arg = bool(arguments.get("include_relations_any_arg")) - include_text_relations_arg1 = arguments.get( - "include_text_relations_arg1", False - ) - predicate_filter = arguments.get("predicate_filter") - limit = arguments.get("limit") - offset = arguments.get("offset") - include_concept_preview = arguments.get("include_concept_preview", True) - include_uncertain = bool(arguments.get("include_uncertain", False)) - uncertainty_mode = arguments.get("uncertainty_mode") - uncertainty_statuses = arguments.get("uncertainty_statuses") - - if predicate_filter is not None and not isinstance(predicate_filter, list): - if isinstance(predicate_filter, (tuple, set)): - predicate_filter = list(predicate_filter) - else: - predicate_filter = [predicate_filter] + vacuous_warning = detect_vacuous_typing(concept) + if vacuous_warning: + concept["_vacuous_typing_warning"] = vacuous_warning - if any( - [ - include_relations_arg1, - include_relations_any_arg, - include_text_relations_arg1, - ] - ): - relations_payload = build_concept_relations_payload( - concept, - include_relations_arg1=include_relations_arg1, - include_relations_any_arg=include_relations_any_arg, - include_text_relations_arg1=include_text_relations_arg1, - predicate_filter=predicate_filter, - limit=limit, - offset=offset, - include_concept_preview=include_concept_preview, - include_uncertain=include_uncertain, - uncertainty_mode=uncertainty_mode, - uncertainty_statuses=uncertainty_statuses, + include_relations_arg1 = bool(arguments.get("include_relations_arg1")) + include_relations_any_arg = bool( + arguments.get("include_relations_any_arg") + ) + include_text_relations_arg1 = arguments.get( + "include_text_relations_arg1", False + ) + predicate_filter = arguments.get("predicate_filter") + limit = arguments.get("limit") + offset = arguments.get("offset") + include_concept_preview = arguments.get( + "include_concept_preview", True ) - concept["relations"] = relations_payload + include_uncertain = bool(arguments.get("include_uncertain", False)) + uncertainty_mode = arguments.get("uncertainty_mode") + uncertainty_statuses = arguments.get("uncertainty_statuses") + + if predicate_filter is not None and not isinstance( + predicate_filter, list + ): + if isinstance(predicate_filter, (tuple, set)): + predicate_filter = list(predicate_filter) + else: + predicate_filter = [predicate_filter] + + if any( + [ + include_relations_arg1, + include_relations_any_arg, + include_text_relations_arg1, + ] + ): + relations_payload = build_concept_relations_payload( + concept, + include_relations_arg1=include_relations_arg1, + include_relations_any_arg=include_relations_any_arg, + include_text_relations_arg1=include_text_relations_arg1, + predicate_filter=predicate_filter, + limit=limit, + offset=offset, + include_concept_preview=include_concept_preview, + include_uncertain=include_uncertain, + uncertainty_mode=uncertainty_mode, + uncertainty_statuses=uncertainty_statuses, + ) + concept["relations"] = relations_payload return [_json_text(concept)] except Exception as exc: return [ @@ -2142,23 +2162,30 @@ async def _handle_find_relations_with_argument( ] try: - payload = find_relations_with_argument( - concept_id=str(concept_id), - argument_index=arguments.get("argument_index"), - predicate_filter=arguments.get("predicate_filter"), - relation_kind=arguments.get("relation_kind"), - scope=arguments.get("scope"), - include_text_snippets=bool(arguments.get("include_text_snippets", False)), - include_concept_preview=bool( - arguments.get("include_concept_preview", True) - ), - limit=arguments.get("limit"), - offset=arguments.get("offset"), - sort_by=arguments.get("sort_by"), - include_uncertain=bool(arguments.get("include_uncertain", False)), - uncertainty_mode=arguments.get("uncertainty_mode"), - uncertainty_statuses=arguments.get("uncertainty_statuses"), + from src.backend.security.access_control import ( + force_access_control_enforcement, ) + + with force_access_control_enforcement(): + payload = find_relations_with_argument( + concept_id=str(concept_id), + argument_index=arguments.get("argument_index"), + predicate_filter=arguments.get("predicate_filter"), + relation_kind=arguments.get("relation_kind"), + scope=arguments.get("scope"), + include_text_snippets=bool( + arguments.get("include_text_snippets", False) + ), + include_concept_preview=bool( + arguments.get("include_concept_preview", True) + ), + limit=arguments.get("limit"), + offset=arguments.get("offset"), + sort_by=arguments.get("sort_by"), + include_uncertain=bool(arguments.get("include_uncertain", False)), + uncertainty_mode=arguments.get("uncertainty_mode"), + uncertainty_statuses=arguments.get("uncertainty_statuses"), + ) return [_json_text(payload)] except Exception as exc: return [ @@ -2306,12 +2333,21 @@ async def _handle_get_text_relations_summary( ] try: - payload = get_text_relations_summary( - concept_id, - predicates=arguments.get("predicates"), - languages=arguments.get("languages"), - max_relation_ids_per_group=arguments.get("max_relation_ids_per_group", 25), + from src.backend.security.access_control import ( + force_access_control_enforcement, ) + + with force_access_control_enforcement(): + payload = get_text_relations_summary( + concept_id, + predicates=arguments.get("predicates"), + languages=arguments.get("languages"), + max_relation_ids_per_group=arguments.get( + "max_relation_ids_per_group", 25 + ), + # Direct stdio calls have no authenticated actor scope. + context_view="base_publication", + ) return [_json_text(payload)] except Exception as exc: return [ @@ -3496,29 +3532,39 @@ async def _handle_search_knowledge_base(arguments: dict[str, Any]) -> list[TextC ) ] try: - service = get_rag_service() - permissions_context = {} - try: - from flask import session as flask_session - - if flask_session.get("user_id"): - permissions_context["user_id"] = flask_session.get("user_id") - if flask_session.get("org_id"): - permissions_context["organisation_concept_id"] = flask_session.get( - "org_id" - ) - except (ImportError, RuntimeError): - pass + from src.backend.security.access_control import ( + get_effective_organisation_concept_id, + get_effective_user_concept_id, + ) - results = service.query( - query_text=query, - top_k=arguments.get("top_k", 5), - namespace=arguments.get("namespace"), - permissions_context=(permissions_context if permissions_context else None), + inherited_preexisting_actor = ( + internal_mcp_gateway_module.get_internal_mcp_preexisting_actor_context() ) - return [ - _json_text({"results": results, "count": len(results), "success": True}) - ] + effective_user_id = get_effective_user_concept_id() + effective_org_id = get_effective_organisation_concept_id() + server_bound_actor = inherited_preexisting_actor or ( + (effective_user_id, effective_org_id) + if effective_user_id or effective_org_id + else None + ) + inherited_source = ( + internal_mcp_gateway_module.get_internal_mcp_actor_context_source() + ) + actor_context_source = ( + "preexisting_authenticated_or_workflow_context" + if server_bound_actor is not None + else "trusted_operator_payload_fallback" + if inherited_source == "trusted_operator_payload_fallback" + else "tool_payload_fallback" + ) + with bind_internal_mcp_actor_context_source( + actor_context_source, + preexisting_actor_context=server_bound_actor, + ): + result = internal_mcp_catalogue_module._search_knowledge_base( + **arguments + ) + return [_json_text(result)] except RAGBackendUnavailable as exc: return [ _json_text({"error": f"RAG service unavailable: {exc}", "success": False}) diff --git a/src/backend/mcp_server/vontology_mcp.json b/src/backend/mcp_server/vontology_mcp.json index 7d665d5ad..630b42817 100644 --- a/src/backend/mcp_server/vontology_mcp.json +++ b/src/backend/mcp_server/vontology_mcp.json @@ -25,19 +25,29 @@ }, { "name": "add_relationship", - "description": "Add one concept-to-concept or concept-to-text relationship. Structural predicates use their existing aliases; a custom predicate uses its exact #V# concept ID. If that exact custom predicate is missing, predicate_if_missing={name, description?} can canonically create or reuse and verify it before this relationship attempt.", + "description": "Add one concept-to-concept or concept-to-text relationship. Structural predicates use their aliases. A custom predicate may be supplied by exact #V# ID or resolved natural-language name. Use predicate_ref for typed ambiguity/not-found recovery and explicitly request creation with on_missing='create_typed_predicate' plus value_kind='concept' or 'text'. Creation/reuse is verified before the relationship attempt.", "inputSchema": { "type": "object", "properties": { "source_id": { "type": "string" }, - "predicate": { - "type": "string" - }, "target": { "type": "string" }, + "predicate": { + "type": [ + "string", + "null" + ] + }, + "predicate_ref": { + "type": [ + "object", + "null" + ], + "additionalProperties": true + }, "predicate_if_missing": { "type": [ "object", @@ -54,11 +64,10 @@ }, "required": [ "source_id", - "predicate", "target" ], "additionalProperties": true, - "description": "add_relationship input: source_id (str), predicate (str, structural relationship name or exact #V# predicate concept ID), target (str). For a missing dynamic predicate only, predicate_if_missing may be {name: str, description?: str}; its name must canonically identify the exact requested predicate. The capability creates or reuses and verifies that predicate before attempting the edge." + "description": "add_relationship input: source_id and target plus exactly one useful predicate reference. predicate accepts a structural relationship, natural-language predicate name, or exact #V# predicate ID. predicate_ref is the typed form: {concept_id} or {name, on_missing?: 'fail'|'create_typed_predicate', value_kind?: 'concept'|'text', description?}. Legacy predicate_if_missing remains supported for exact dynamic predicates. Creation is attempted only when explicitly requested, then verified before the edge." } }, { @@ -202,14 +211,14 @@ "null" ] }, + "include_legacy": { + "type": "boolean" + }, "offset": { "type": "integer" }, "limit": { "type": "integer" - }, - "include_legacy": { - "type": "boolean" } }, "required": [], @@ -961,7 +970,7 @@ "concept_id" ], "additionalProperties": true, - "description": "delete_text_relation input: concept_id (str), relation_id (str, optional - preferred method), predicate (str, optional for predicate+text deletion), text (str, optional with predicate), language (str, optional), garbage_collect (bool, optional - if true, delete orphaned text_values)" + "description": "delete_text_relation input: concept_id plus a base_text_relation relation_id (preferred), or predicate+text and optional language. Scoped assertion IDs are not text-relation IDs. garbage_collect optionally deletes orphaned text_values." } }, { @@ -1943,7 +1952,7 @@ }, { "name": "fetch_concept", - "description": "Fetch full details of ONE specific concept by its ID (format: #V#concept_name). Use when you already know the exact concept_id and need complete information (description, predicates, relationships). Don't use for searching.", + "description": "Fetch full details of one concept by exact ID. An authenticated actor receives actor_effective scoped enrichment; identity or namespace supplied only in an ordinary tool payload is ignored and the response is explicitly base_publication. Don't use for searching.", "inputSchema": { "type": "object", "properties": { @@ -2102,7 +2111,7 @@ }, { "name": "find_relations_with_argument", - "description": "Find relation assertions where a concept appears in one or more argument positions. The anchor entity must be supplied in concept_id; use argument_index to choose the relation slot instead of inventing subject/object payload fields. Supports exact graph matching and full-text text-relation matching with optional predicate/kind filters and pagination.", + "description": "Find relation assertions where a concept appears in one or more argument positions. The anchor entity must be supplied in concept_id; use argument_index to choose the relation slot instead of inventing subject/object payload fields. Supports exact graph matching and full-text text-relation matching with optional predicate/kind filters and pagination. Verified actors receive actor_effective results; payload-only identity is ignored and yields base_publication. Truncated actor overlays expose bounded recovery rather than a misleading offset continuation.", "inputSchema": { "type": "object", "properties": { @@ -2305,7 +2314,7 @@ }, { "name": "get_predicate_incidence", - "description": "Summarise which predicates are actually observed around a concept or across instances of a type. The anchor entity goes in concept_id; argument_index='subject' means inspect outgoing subject-side relations, not that the payload should contain a subject field. Use when you need distinct predicates plus counts before choosing a predicate-specific extent or filtered relation lookup. For kind-specific turns such as papers, projects, students, organisations, or other represented related things, set include_argument_type_counts=true to get direct asserted type distributions for non-anchor arguments. Set role_expansion_mode='explicit' with represented node-type or role-predicate filters when the immediate neighbour is a reified/event/claim node whose other role fillers are the useful retrieval targets.", + "description": "Summarise which predicates are actually observed around a concept or across instances of a type. The anchor entity goes in concept_id; argument_index='subject' means inspect outgoing subject-side relations, not that the payload should contain a subject field. Use when you need distinct predicates plus counts before choosing a predicate-specific extent or filtered relation lookup. Start with bounded minimal incidence (a modest limit, without argument type counts, previews, or text snippets), then request those richer fields only when the initial evidence shows they are needed; avoid concurrent rich incidence probes by default. Set role_expansion_mode='explicit' with represented node-type or role-predicate filters when the immediate neighbour is a reified/event/claim node whose other role fillers are the useful retrieval targets.", "inputSchema": { "type": "object", "properties": { @@ -2557,7 +2566,7 @@ }, { "name": "get_text_relations", - "description": "Retrieve text relations for a concept, optionally filtered by predicate/language. Returns all text attachments (hasContent, hasDescription, hasName, etc.). Use to query what text is attached to a concept.", + "description": "Retrieve labelled text-assertion rows for a concept, optionally filtered by predicate/language. The serving authority boundary selects context_view: trusted actor-bound internal calls use actor_effective; actorless calls and ordinary payload-only identity use base_publication. Inspect row_kind; only a base_text_relation relation_id is valid for update/delete.", "inputSchema": { "type": "object", "properties": { @@ -2592,7 +2601,7 @@ }, { "name": "get_text_relations_summary", - "description": "Return a lightweight summary of text relations for a concept: counts + relation IDs grouped by predicate/language (no full text bodies). Use to quickly decide what to fetch next.", + "description": "Return a lightweight, context-labelled text-assertion summary grouped by predicate/language (no full text bodies). The serving authority boundary selects actor_effective or base_publication; ordinary payload-only identity is ignored. Relation and assertion IDs remain separate; inspect counts_are_lower_bounds before treating scoped counts as complete.", "inputSchema": { "type": "object", "properties": { @@ -6136,7 +6145,7 @@ "new_text" ], "additionalProperties": true, - "description": "update_text_relation input: concept_id (str), relation_id (str), new_text (str), language (str, optional)" + "description": "update_text_relation input: concept_id, relation_id, new_text, and optional language. relation_id must come from a base_text_relation row; scoped_assertion IDs are not mutable text-relation IDs." } }, { @@ -6249,7 +6258,7 @@ }, { "name": "upsert_text_relation", - "description": "Add or update ANY text relation (hasContent, hasDescription, hasNote, custom predicates, etc.). Use for attaching text content to concepts with flexible predicate types. More general than add_names which is specialized for hasName relations only.", + "description": "Add or update a canonical text relation when the actor has mutation authority over the subject concept. On an actor-bound surface that exposes upsert_scoped_assertion, use that capability for knowledge about a visible concept the actor does not own. Supports hasContent, hasDescription, hasNote, hasName, and existing custom predicate concepts.", "inputSchema": { "type": "object", "properties": { @@ -6741,10 +6750,16 @@ "type": "string" }, "user_id": { - "type": "string" + "type": [ + "string", + "null" + ] }, "org_id": { - "type": "string" + "type": [ + "string", + "null" + ] }, "namespace": { "type": [ diff --git a/src/backend/security/access_control.py b/src/backend/security/access_control.py index 051c96da7..52f770fb2 100644 --- a/src/backend/security/access_control.py +++ b/src/backend/security/access_control.py @@ -25,6 +25,10 @@ _MANUAL_USER: ContextVar[Optional[str]] = ContextVar("access_manual_user", default=None) _MANUAL_ORG: ContextVar[Optional[str]] = ContextVar("access_manual_org", default=None) _BYPASS: ContextVar[bool] = ContextVar("access_bypass", default=False) +_FORCE_ENFORCEMENT: ContextVar[bool] = ContextVar( + "access_force_enforcement", + default=False, +) _EVALUATOR: ContextVar["AccessEvaluator | None"] = ContextVar( "access_evaluator", default=None ) @@ -425,6 +429,8 @@ def is_bypass_enabled() -> bool: def should_enforce_access_control() -> bool: if is_bypass_enabled(): return False + if _FORCE_ENFORCEMENT.get(): + return True if _MANUAL_USER.get() is not None or _MANUAL_ORG.get() is not None: return True return has_request_context() @@ -876,6 +882,19 @@ def override_current_actor( _MANUAL_USER.reset(user_token) +@contextmanager +def force_access_control_enforcement(): + """Enforce global-only visibility even when no actor identity is bound.""" + + token = _FORCE_ENFORCEMENT.set(True) + eval_token = _EVALUATOR.set(None) + try: + yield + finally: + _EVALUATOR.reset(eval_token) + _FORCE_ENFORCEMENT.reset(token) + + @contextmanager def bypass_access_control(): token = _BYPASS.set(True) diff --git a/src/backend/services/adaptive_turn_service.py b/src/backend/services/adaptive_turn_service.py index f0a95edef..84ac55765 100644 --- a/src/backend/services/adaptive_turn_service.py +++ b/src/backend/services/adaptive_turn_service.py @@ -1875,6 +1875,105 @@ def _error_payload( } +def _exact_represented_concept_id(value: Any) -> str | None: + """Return an explicit represented ID without resolving or canonicalising it.""" + + if not isinstance(value, str): + return None + cleaned = value.strip() + if ( + not cleaned.startswith("#V#") + or len(cleaned) == len("#V#") + or any(character.isspace() for character in cleaned) + ): + return None + return cleaned + + +def _exact_scoped_relationship_predicate( + arguments: Mapping[str, Any], +) -> str | None: + """Extract only a concept-valued predicate that needs no semantic resolution.""" + + if arguments.get("predicate_if_missing") is not None: + return None + + predicate = arguments.get("predicate") + predicate_ref = arguments.get("predicate_ref") + if isinstance(predicate, str) and predicate.strip(): + if predicate_ref is not None: + return None + return _exact_represented_concept_id(predicate) + + if not isinstance(predicate_ref, Mapping): + return None + if isinstance(predicate_ref.get("name"), str) and predicate_ref["name"].strip(): + return None + if str(predicate_ref.get("on_missing") or "fail").strip().lower() != "fail": + return None + if str(predicate_ref.get("value_kind") or "concept").strip().lower() != "concept": + return None + return _exact_represented_concept_id(predicate_ref.get("concept_id")) + + +def _effect_subject_authority_denial( + *, + capability_name: str, + arguments: Mapping[str, Any], + scoped_assertion_available: bool, +) -> dict[str, Any]: + """Return a bounded denial while preserving a semantically distinct path.""" + + payload = _error_payload( + "effect_subject_not_authorised", + "The effect subject is not scoped to the authenticated actor or organisation.", + ) + if not scoped_assertion_available: + return payload + + if capability_name == "upsert_text_relation": + alternative_arguments = { + "subject_concept_id": arguments.get("concept_id"), + "predicate": arguments.get("predicate"), + "target_text": arguments.get("text"), + } + language = arguments.get("language") + if isinstance(language, str) and language.strip(): + alternative_arguments["language"] = language + semantic_effect = ( + "if the subject is visible, create a provenance-bearing " + "actor-scoped assertion without publishing or mutating it" + ) + elif capability_name == "add_relationship": + source_id = _exact_represented_concept_id(arguments.get("source_id")) + target_id = _exact_represented_concept_id(arguments.get("target")) + predicate_id = _exact_scoped_relationship_predicate(arguments) + if source_id is None or predicate_id is None or target_id is None: + return payload + alternative_arguments = { + "subject_concept_id": source_id, + "predicate": predicate_id, + "target_concept_id": target_id, + } + semantic_effect = ( + "if the subject, predicate, and target are visible, create a " + "provenance-bearing actor-scoped assertion without publishing " + "or mutating the canonical relationship" + ) + else: + return payload + + payload["recovery_affordances"] = [ + { + "action_type": "assert_in_actor_scope", + "tool": "upsert_scoped_assertion", + "arguments": alternative_arguments, + "semantic_effect": semantic_effect, + } + ] + return payload + + def _emit(progress_tracker: Any, payload: Mapping[str, Any]) -> None: if progress_tracker is None: return @@ -2037,6 +2136,117 @@ def execute_adaptive_turn( last_partial_effect_generation = 0 successful_effect_mutation_generation = 0 terminal_failed_effect_requests: dict[str, tuple[int, str | None]] = {} + recoverable_effect_ids: dict[str, list[str]] = {} + + def scoped_assertion_recovery_key( + capability_name: str, + arguments: Mapping[str, Any], + ) -> str | None: + """Identify one exact scoped assertion independently of trusted bindings.""" + + if capability_name != "upsert_scoped_assertion": + return None + subject_id = str(arguments.get("subject_concept_id") or "").strip() + predicate = str(arguments.get("predicate") or "").strip() + if not subject_id or not predicate: + return None + try: + from .text_relation_predicate_validation_service import ( + predicate_concept_id_for_storage, + ) + + predicate = predicate_concept_id_for_storage(predicate) or predicate + except Exception: + pass + target_text = arguments.get("target_text") + target_concept_id = str( + arguments.get("target_concept_id") or "" + ).strip() + has_text = isinstance(target_text, str) and bool(target_text.strip()) + has_concept = bool(target_concept_id) + if has_text == has_concept: + return None + semantic_identity = { + "subject_concept_id": subject_id, + "predicate": predicate, + "target_kind": "text" if has_text else "concept", + "target": ( + str(target_text).strip() if has_text else target_concept_id + ), + "language": ( + str(arguments.get("language") or "en-NZ").strip() or "en-NZ" + if has_text + else None + ), + "scope_mode": ( + str(arguments.get("scope_mode") or "user").strip().lower() + or "user" + ), + } + return hashlib.sha256(_json_bytes(semantic_identity)).hexdigest() + + def remember_recovery_affordance( + *, + effect_id: str, + payload: Any, + ) -> None: + if not isinstance(payload, Mapping): + return + affordances = payload.get("recovery_affordances") + if not isinstance(affordances, Sequence) or isinstance( + affordances, + (str, bytes, bytearray), + ): + return + for affordance in affordances: + if not isinstance(affordance, Mapping): + continue + tool_name = str(affordance.get("tool") or "").strip() + alternative_arguments = affordance.get("arguments") + if not isinstance(alternative_arguments, Mapping): + continue + recovery_key = scoped_assertion_recovery_key( + tool_name, + alternative_arguments, + ) + if recovery_key is None: + continue + recoverable_effect_ids.setdefault(recovery_key, []).append( + effect_id + ) + + def reconcile_successful_recovery( + *, + recovery_effect_id: str, + capability_name: str, + arguments: Mapping[str, Any], + effect_status: str, + ) -> None: + nonlocal effect_state_generation + if effect_status != "succeeded": + return + recovery_key = scoped_assertion_recovery_key( + capability_name, + arguments, + ) + if recovery_key is None: + return + pending_effect_ids = recoverable_effect_ids.get(recovery_key) + if not pending_effect_ids: + return + failed_effect_id = pending_effect_ids.pop(0) + if not pending_effect_ids: + recoverable_effect_ids.pop(recovery_key, None) + with effect_state_lock: + failed_state = effect_states.get(failed_effect_id) + if ( + failed_state is None + or failed_state.get("effect_status") != "failed" + ): + return + failed_state["recovered_by_effect_id"] = recovery_effect_id + failed_state["recovery_status"] = "succeeded" + effect_state_generation += 1 def remember_effect_state( effect_id: str, @@ -2076,6 +2286,13 @@ def remember_effect_state( ) if key in late_observation } + if existing is not None: + for recovery_field in ( + "recovered_by_effect_id", + "recovery_status", + ): + if recovery_field in existing: + state[recovery_field] = existing[recovery_field] effect_states[effect_id] = state effect_state_generation += 1 @@ -2211,6 +2428,11 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: for state in effect_snapshot.values() if state.get("effect_status") in {"failed", "partial", "indeterminate", "not_started"} + and not ( + state.get("effect_status") == "failed" + and state.get("recovery_status") == "succeeded" + and state.get("recovered_by_effect_id") + ) ] if status == "completed" and incomplete_effects: incomplete_statuses = { @@ -2238,6 +2460,13 @@ def finish(text: str, *, status: str = "completed") -> AdaptiveTurnResult: invocation["execution_id"] = state.get("execution_id") if state.get("evidence_id"): invocation["late_evidence_id"] = state.get("evidence_id") + if state.get("recovered_by_effect_id"): + invocation["recovered_by_effect_id"] = state.get( + "recovered_by_effect_id" + ) + invocation["recovery_status"] = state.get( + "recovery_status" + ) if isinstance(state.get("late_observation"), Mapping): invocation["late_completion"] = dict(state["late_observation"]) reconciled_invocations.append(invocation) @@ -3286,10 +3515,15 @@ def contained( scope, ): return index, contained( - _error_payload( - "effect_subject_not_authorised", - "The effect subject is not scoped to the authenticated " - "actor or organisation.", + _effect_subject_authority_denial( + capability_name=canonical_name, + arguments=arguments, + scoped_assertion_available=( + gateway.get_method_definition( + "upsert_scoped_assertion" + ) + is not None + ), ) ) @@ -3746,6 +3980,16 @@ def contained( or None ), ) + remember_recovery_affordance( + effect_id=effect_identifier, + payload=raw_payload, + ) + reconcile_successful_recovery( + recovery_effect_id=effect_identifier, + capability_name=canonical_name, + arguments=arguments, + effect_status=effect_status, + ) envelope_payload.update( { "effect_id": effect_identifier, diff --git a/src/backend/services/concept_relation_service.py b/src/backend/services/concept_relation_service.py index fb24662ae..234266ce0 100644 --- a/src/backend/services/concept_relation_service.py +++ b/src/backend/services/concept_relation_service.py @@ -5,7 +5,18 @@ import re from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) from ..db.repositories.concepts_repository import ConceptsRepository from ..db.repositories.text_value_repository import ( @@ -30,6 +41,8 @@ from .text_value_service import get_texts_for_concept, get_texts_for_concepts RelationValue = Dict[str, Any] +RelationContextView = Literal["base_publication", "actor_effective"] +_RELATION_CONTEXT_VIEWS = frozenset({"base_publication", "actor_effective"}) _ARG_INDEX_SUBJECT = 1 # Align with example payloads (1-based indexing) _ARG_INDEX_FIRST_OBJECT = 2 @@ -293,6 +306,7 @@ def find_relations_with_argument( include_uncertain: bool = False, uncertainty_mode: Optional[str] = None, uncertainty_statuses: Optional[Sequence[str]] = None, + context_view: RelationContextView = "base_publication", ) -> Dict[str, Any]: """Find relation hits where ``concept_id`` appears in any argument position. @@ -305,6 +319,10 @@ def find_relations_with_argument( resolved_concept_id = str(concept_id or "").strip() if not resolved_concept_id: raise ValueError("concept_id is required") + if context_view not in _RELATION_CONTEXT_VIEWS: + raise ValueError( + "context_view must be 'base_publication' or 'actor_effective'" + ) _ = scope # Scope is enforced by repository-level access control. resolved_limit = _coerce_limit(limit) @@ -329,6 +347,7 @@ def find_relations_with_argument( "sort_by": sort_by, "include_text_snippets": bool(include_text_snippets), "include_concept_preview": bool(include_concept_preview), + "context_view": context_view, } include_structural = relation_filter in {"any", "binary"} @@ -360,6 +379,8 @@ def find_relations_with_argument( preview_cache: Dict[str, Optional[Dict[str, Any]]] = {} hits: List[Dict[str, Any]] = [] + scoped_concept_query_truncated = False + actor_effective_text_query_truncated = False subject_doc = _load_accessible_relation_subject_document( resolved_concept_id, @@ -645,15 +666,28 @@ def find_relations_with_argument( hit["source_concept_preview"] = source_preview hits.append(hit) - if include_asserted_rows and include_structural: - from .scoped_assertion_service import list_visible_scoped_assertions + if ( + context_view == "actor_effective" + and include_asserted_rows + and include_structural + ): + from .scoped_assertion_service import ( + list_visible_scoped_assertions_page, + ) - for assertion in list_visible_scoped_assertions( + scoped_concept_page = list_visible_scoped_assertions_page( argument_concept_id=resolved_concept_id, predicates=predicate_display_terms or None, object_kind="concept", - limit=_MAX_LIMIT, - ): + limit=_MAX_LIMIT + 1, + ) + scoped_concept_assertions = scoped_concept_page["items"] + scoped_concept_query_truncated = bool( + scoped_concept_page.get("has_more") + or scoped_concept_page.get("counts_are_lower_bounds") + or len(scoped_concept_assertions) > _MAX_LIMIT + ) + for assertion in scoped_concept_assertions[:_MAX_LIMIT]: source_id = str(assertion.get("subject_concept_id") or "") target_id = str(assertion.get("object_concept_id") or "") predicate_id = assertion.get("predicate") @@ -689,8 +723,9 @@ def find_relations_with_argument( else None ), "relation_metadata": { - "relation_id": assertion.get("assertion_id"), + "relation_id": None, "assertion_id": assertion.get("assertion_id"), + "row_kind": "scoped_assertion", "updated_at": assertion.get("updated_at"), "match_type": "exact", "assertion_scope": assertion.get("scope"), @@ -719,14 +754,31 @@ def find_relations_with_argument( include_concept_preview, preview_cache, ) - for rel in get_texts_for_concept( - subject_concept_id=resolved_concept_id, - limit=_MAX_LIMIT, - ): + text_query_metadata: dict[str, Any] = {} + actor_effective_text_rows = get_texts_for_concepts( + [resolved_concept_id], + limit_per_concept=_MAX_LIMIT + 1, + context_view=context_view, + query_metadata=text_query_metadata, + ).get(resolved_concept_id, []) + actor_effective_text_query_truncated = bool( + len(actor_effective_text_rows) > _MAX_LIMIT + or text_query_metadata.get("relation_query_truncated") + or text_query_metadata.get("scoped_query_truncated") + or text_query_metadata.get("scoped_counts_are_lower_bounds") + ) + for rel in actor_effective_text_rows[:_MAX_LIMIT]: predicate_id = rel.get("predicate") if not _predicate_matches_terms(predicate_id, predicate_terms): continue text_value = rel.get("text") + assertion_id = rel.get("assertion_id") + row_kind = rel.get("row_kind") + relation_id = rel.get("relation_id") + if row_kind == "scoped_assertion" or assertion_id: + relation_id = None + elif not relation_id: + relation_id = f"text::{resolved_concept_id}::{predicate_id}" hit: Dict[str, Any] = { "source_concept_id": resolved_concept_id, "predicate_concept_id": predicate_id, @@ -734,14 +786,14 @@ def find_relations_with_argument( "argument_indexes": [_ARG_INDEX_SUBJECT], "target_value": text_value, "relation_metadata": { - "relation_id": str( - rel.get("relation_id") - or f"text::{resolved_concept_id}::{predicate_id}" + "relation_id": ( + str(relation_id) if relation_id is not None else None ), "text_value_id": rel.get("text_value_id"), "lang": rel.get("lang"), "match_type": "exact", - "assertion_id": rel.get("assertion_id"), + "assertion_id": assertion_id, + "row_kind": row_kind, "assertion_scope": rel.get("assertion_scope"), "canonical_publication": rel.get( "canonical_publication", @@ -883,15 +935,29 @@ def find_relations_with_argument( deduped_hits = _dedupe_argument_hits(hits) sorted_hits = _sort_argument_hits(deduped_hits, sort_by) paged_hits = sorted_hits[resolved_offset : resolved_offset + resolved_limit_value] + counts_are_lower_bounds = bool( + scoped_concept_query_truncated or actor_effective_text_query_truncated + ) return { "concept_id": resolved_concept_id, + "context_view": context_view, "total_hits": len(sorted_hits), + **( + {"total_hits_is_lower_bound": True} + if counts_are_lower_bounds + else {} + ), "hits": paged_hits, "paging": { "limit": resolved_limit_value, "offset": resolved_offset, "returned": len(paged_hits), "total_available": len(sorted_hits), + **( + {"total_available_is_lower_bound": True} + if counts_are_lower_bounds + else {} + ), }, "uncertainty_diagnostics": { "mode": mode_value, @@ -905,6 +971,19 @@ def find_relations_with_argument( "uncertainty_statuses": status_filter or None, "total_hits": len(sorted_hits), "returned": len(paged_hits), + **( + { + "total_hits_is_lower_bound": True, + "scoped_concept_query_truncated": ( + scoped_concept_query_truncated + ), + "actor_effective_text_query_truncated": ( + actor_effective_text_query_truncated + ), + } + if counts_are_lower_bounds + else {} + ), "incoming_asserted_binary": incoming_asserted_binary_diagnostics, }, } @@ -3717,6 +3796,7 @@ def _dedupe_argument_hits(hits: Sequence[Dict[str, Any]]) -> List[Dict[str, Any] tuple(hit.get("argument_indexes") or []), hit.get("target_value"), metadata.get("relation_id"), + metadata.get("assertion_id"), ) existing = deduped.get(key) if existing is None: diff --git a/src/backend/services/rag_backends/llamaindex_backend.py b/src/backend/services/rag_backends/llamaindex_backend.py index c405ebfe5..c3800315d 100644 --- a/src/backend/services/rag_backends/llamaindex_backend.py +++ b/src/backend/services/rag_backends/llamaindex_backend.py @@ -1195,11 +1195,97 @@ def query( for attr, value in previous_embed_settings.items(): setattr(embed_model, attr, value) + knowledge_candidate_metadata = [ + metadata + for node in nodes + if isinstance( + metadata := getattr(node.node, "metadata", None), + dict, + ) + and ( + metadata.get("type") + in {"text_relation", "scoped_knowledge_assertion"} + or metadata.get("source") + in { + "vontology_text_relation", + "scoped_knowledge_assertion", + } + ) + ] + authorised_knowledge_keys: set[tuple[str, str]] = set() + if knowledge_candidate_metadata: + from ..scoped_rag_authority_service import ( + current_authorised_rag_candidate_keys, + ) + + actor_user_id = ( + permissions_context.get("user_id") + if isinstance(permissions_context, dict) + else None + ) + actor_org_id = ( + permissions_context.get("organisation_concept_id") + if isinstance(permissions_context, dict) + else None + ) + try: + authorised_knowledge_keys = ( + current_authorised_rag_candidate_keys( + knowledge_candidate_metadata, + user_concept_id=actor_user_id, + organisation_concept_id=actor_org_id, + ) + ) + except Exception as exc: + # Knowledge rows fail closed when their live authority surface + # is unavailable; unrelated chat candidates remain usable. + logger.warning( + "Knowledge RAG authority refresh failed closed: %s", + exc, + ) + authorised_knowledge_keys = set() + + # Live authority is part of the retrieval boundary, not an ordinary + # semantic filter. Remove inaccessible knowledge rows before computing + # any externally visible count or candidate-window diagnostic so a + # stale vector entry cannot disclose that a relation or assertion exists. + authority_visible_nodes = [] + for node in nodes: + metadata = getattr(node.node, "metadata", None) + candidate_key: tuple[str, str] | None = None + if isinstance(metadata, dict): + metadata_type = str(metadata.get("type") or "").strip() + metadata_source = str(metadata.get("source") or "").strip() + if ( + metadata_type == "scoped_knowledge_assertion" + or metadata_source == "scoped_knowledge_assertion" + ): + candidate_key = ( + "scoped_knowledge_assertion", + str(metadata.get("assertion_id") or "").strip(), + ) + elif ( + metadata_type == "text_relation" + or metadata_source == "vontology_text_relation" + ): + candidate_key = ( + "text_relation", + str(metadata.get("relation_id") or "").strip(), + ) + if candidate_key is not None: + if ( + not candidate_key[1] + or candidate_key not in authorised_knowledge_keys + ): + continue + authority_visible_nodes.append(node) + def _matches_permissions(metadata: Any) -> bool: - if not permissions_context: - return True if not isinstance(metadata, dict): return False + actual_type = metadata.get("type") + if not permissions_context: + return True # Optional semantic filtering. # Backwards compatible: if no filter keys provided, behaviour is unchanged. @@ -1218,12 +1304,14 @@ def _matches_permissions(metadata: Any) -> bool: if m == "chat": requested_types = ["chat_message"] elif m == "concepts": - requested_types = ["text_relation"] + requested_types = [ + "text_relation", + "scoped_knowledge_assertion", + ] elif m == "all" or not m: pass if requested_types: - actual_type = metadata.get("type") # Defensive fallback for older indices missing explicit type. if not isinstance(actual_type, str) or not actual_type.strip(): src = metadata.get("source") @@ -1273,10 +1361,8 @@ def _matches_permissions(metadata: Any) -> bool: return True results = [] - filtered_candidate_count = 0 - for node in nodes: + for node in authority_visible_nodes: if not _matches_permissions(getattr(node.node, "metadata", None)): - filtered_candidate_count += 1 continue results.append( { @@ -1291,55 +1377,32 @@ def _matches_permissions(metadata: Any) -> bool: break elapsed_ms = int((time.perf_counter() - start) * 1000) - filter_keys = { - "mode", - "type", - "predicate", - "predicates", - "user_id", - "organisation_concept_id", - } - local_filter_requested = bool( - isinstance(permissions_context, dict) - and any(permissions_context.get(key) not in (None, "", [], {}) for key in filter_keys) - ) - candidate_count = len(nodes) if isinstance(nodes, list) else None - candidate_limit_reached = bool( - candidate_count is not None and candidate_count >= similarity_top_k - ) - filtered_window_incomplete = bool( - local_filter_requested - and filtered_candidate_count - and candidate_limit_reached - and len(results) < max(1, int(top_k)) - ) - filtered_window_exhausted = bool( - filtered_window_incomplete and not results - ) - filtered_window_partial = bool(filtered_window_incomplete and results) + requested_result_count = max(1, int(top_k)) + visible_result_count = len(results) + visible_window_complete = visible_result_count >= requested_result_count query_results = self._query_results_with_state( results, namespace=effective_namespace, status=( "results_available" - if results and not filtered_window_partial + if visible_window_complete else "partial_results" - if filtered_window_partial - else "candidate_window_exhausted" - if filtered_window_exhausted - else "valid_empty" + if visible_result_count + else "degraded" ), cause=( - "local_filter_candidate_window_exhausted" - if filtered_window_exhausted - else "local_filter_candidate_window_partial" - if filtered_window_partial - else "query_completed" + "query_completed" + if visible_window_complete + else "bounded_visible_result_window_underfilled" + if visible_result_count + else "bounded_visible_result_window_inconclusive" ), - candidate_count=candidate_count, - filtered_candidate_count=filtered_candidate_count, - candidate_limit=similarity_top_k, - candidate_limit_reached=candidate_limit_reached, + # Public diagnostics are deliberately authority-opaque. Expose only + # the visible result count and requested result bound, never raw, + # permission-filtered, or authority-filtered candidate volumes. + candidate_count=visible_result_count, + candidate_limit=requested_result_count, + candidate_limit_reached=visible_window_complete, ) try: self._last_query_info = { @@ -1353,7 +1416,7 @@ def _matches_permissions(metadata: Any) -> bool: if retrieval_candidate_limit is not None else None ), - "retrieved": len(nodes) if isinstance(nodes, list) else None, + "retrieved": visible_result_count, "returned": len(results), "elapsed_ms": elapsed_ms, "retrieval_state": dict(query_results.retrieval_state), diff --git a/src/backend/services/rag_text_relation_change_hook_service.py b/src/backend/services/rag_text_relation_change_hook_service.py index dc696a564..1f8fbe970 100644 --- a/src/backend/services/rag_text_relation_change_hook_service.py +++ b/src/backend/services/rag_text_relation_change_hook_service.py @@ -1,55 +1,237 @@ from __future__ import annotations import logging +import queue +import threading from typing import Any, Dict, Optional logger = logging.getLogger(__name__) +_RAG_SYNC_QUEUE_MAX_SIZE = 256 +_RAG_SYNC_QUEUE: queue.Queue[dict[str, str | None]] = queue.Queue( + maxsize=_RAG_SYNC_QUEUE_MAX_SIZE +) +_RAG_SYNC_LOCK = threading.Lock() +_RAG_SYNC_PENDING_KEYS: set[tuple[str, str, str | None, str | None]] = set() +_RAG_SYNC_RERUN_KEYS: set[tuple[str, str, str | None, str | None]] = set() +_RAG_SYNC_WORKER_THREAD: threading.Thread | None = None +_QUEUE_RECEIPT_FIELDS = { + "mechanism": "in_memory_bounded_queue", + "durable": False, +} + + +def _run_concept_text_relation_rag_sync( + *, + namespace: str, + concept_id: str, + predicate: str | None, + scoped_assertion_id: str | None, +) -> Dict[str, Any]: + """Run one synchronous RAG refresh outside the primary write path.""" + + if scoped_assertion_id: + from src.backend.services.rag_text_relation_sync_service import ( + sync_scoped_assertions_to_rag, + ) + + return sync_scoped_assertions_to_rag( + namespace=namespace, + assertion_ids=[scoped_assertion_id], + ) + + from src.backend.services.rag_text_relation_sync_service import ( + sync_text_relations_to_rag, + ) + return sync_text_relations_to_rag( + namespace=namespace, + concept_ids=[concept_id], + predicates=[predicate] if predicate else None, + limit=5000, + ) + + +def _concept_text_relation_rag_sync_worker() -> None: + while True: + job = _RAG_SYNC_QUEUE.get() + namespace = str(job["namespace"]) + concept_id = str(job["concept_id"]) + predicate_value = job.get("predicate") + predicate = str(predicate_value) if predicate_value else None + scoped_assertion_id_value = job.get("scoped_assertion_id") + scoped_assertion_id = ( + str(scoped_assertion_id_value) if scoped_assertion_id_value else None + ) + pending_key = (namespace, concept_id, predicate, scoped_assertion_id) + try: + result = _run_concept_text_relation_rag_sync( + namespace=namespace, + concept_id=concept_id, + predicate=predicate, + scoped_assertion_id=scoped_assertion_id, + ) + if not result.get("success", False): + logger.warning( + "[rag_text_relations] Background RAG sync did not succeed " + "concept_id=%s namespace=%s result=%s", + concept_id, + namespace, + result, + ) + except Exception as exc: # pragma: no cover - defensive worker boundary + logger.warning( + "[rag_text_relations] Background RAG sync failed " + "concept_id=%s namespace=%s: %s", + concept_id, + namespace, + exc, + ) + finally: + with _RAG_SYNC_LOCK: + if pending_key in _RAG_SYNC_RERUN_KEYS: + _RAG_SYNC_RERUN_KEYS.discard(pending_key) + try: + _RAG_SYNC_QUEUE.put_nowait(job) + except queue.Full: # pragma: no cover - worker drained one slot + _RAG_SYNC_PENDING_KEYS.discard(pending_key) + logger.warning( + "[rag_text_relations] Could not queue requested " + "follow-up refresh concept_id=%s namespace=%s", + concept_id, + namespace, + ) + else: + _RAG_SYNC_PENDING_KEYS.discard(pending_key) + _RAG_SYNC_QUEUE.task_done() + def maybe_sync_concept_text_relations_to_rag( *, namespace: Optional[str], concept_id: str, predicate: Optional[str] = None, + scoped_assertion_id: Optional[str] = None, ) -> Dict[str, Any]: - """Best-effort: reindex a concept's text relations into RAG. + """Queue a best-effort concept RAG refresh outside the primary write path. - This is intentionally fail-closed: if `namespace` is missing, we do nothing. + The bounded queue prevents derived index maintenance from delaying or + changing the receipt for the durable Vontology write. Exact duplicate work + is coalesced while pending. Queue saturation remains an observable, + recoverable indexing miss rather than backpressure on the primary mutation. """ if not namespace or not isinstance(namespace, str) or not namespace.strip(): - return {"success": False, "skipped": True, "reason": "namespace_required"} + return { + **_QUEUE_RECEIPT_FIELDS, + "success": False, + "scheduled": False, + "skipped": True, + "reason": "namespace_required", + } if not isinstance(concept_id, str) or not concept_id.strip(): - return {"success": False, "skipped": True, "reason": "concept_id_required"} + return { + **_QUEUE_RECEIPT_FIELDS, + "success": False, + "scheduled": False, + "skipped": True, + "reason": "concept_id_required", + } - predicates = None + clean_predicate = None if isinstance(predicate, str) and predicate.strip(): - predicates = [predicate.strip()] + clean_predicate = predicate.strip() + clean_namespace = namespace.strip() + clean_concept_id = concept_id.strip() + clean_scoped_assertion_id = ( + scoped_assertion_id.strip() + if isinstance(scoped_assertion_id, str) + and scoped_assertion_id.strip().startswith("ska_") + else None + ) + pending_key = ( + clean_namespace, + clean_concept_id, + clean_predicate, + clean_scoped_assertion_id, + ) + job = { + "namespace": clean_namespace, + "concept_id": clean_concept_id, + "predicate": clean_predicate, + "scoped_assertion_id": clean_scoped_assertion_id, + } + global _RAG_SYNC_WORKER_THREAD try: - from src.backend.services.rag_text_relation_sync_service import ( - sync_text_relations_to_rag, - ) + with _RAG_SYNC_LOCK: + if pending_key in _RAG_SYNC_PENDING_KEYS: + # A refresh already running may have read before this write. + # Mark it dirty so the worker performs one follow-up refresh + # after the current pass rather than losing the newer state. + _RAG_SYNC_RERUN_KEYS.add(pending_key) + return { + **_QUEUE_RECEIPT_FIELDS, + "success": True, + "scheduled": False, + "reason": "already_scheduled", + "rerun_requested": True, + } + if ( + _RAG_SYNC_WORKER_THREAD is None + or not _RAG_SYNC_WORKER_THREAD.is_alive() + ): + worker = threading.Thread( + target=_concept_text_relation_rag_sync_worker, + name="rag-text-relation-sync", + daemon=True, + ) + worker.start() + _RAG_SYNC_WORKER_THREAD = worker - return sync_text_relations_to_rag( - namespace=namespace.strip(), - concept_ids=[concept_id.strip()], - predicates=predicates, - limit=5000, - ) - except Exception as exc: + _RAG_SYNC_PENDING_KEYS.add(pending_key) + try: + _RAG_SYNC_QUEUE.put_nowait(job) + except queue.Full: + _RAG_SYNC_PENDING_KEYS.discard(pending_key) + _RAG_SYNC_RERUN_KEYS.discard(pending_key) + logger.warning( + "[rag_text_relations] Background RAG sync queue full; " + "concept_id=%s namespace=%s remains manually reindexable", + clean_concept_id, + clean_namespace, + ) + return { + **_QUEUE_RECEIPT_FIELDS, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "queue_full", + } + except Exception as exc: # primary-write receipt must not depend on maintenance + with _RAG_SYNC_LOCK: + _RAG_SYNC_PENDING_KEYS.discard(pending_key) + _RAG_SYNC_RERUN_KEYS.discard(pending_key) logger.warning( - "[rag_text_relations] RAG sync failed concept_id=%s namespace=%s: %s", - concept_id, - namespace, + "[rag_text_relations] Could not schedule background RAG sync " + "concept_id=%s namespace=%s: %s", + clean_concept_id, + clean_namespace, exc, ) return { + **_QUEUE_RECEIPT_FIELDS, "success": False, + "scheduled": False, "skipped": False, - "error": f"RAG sync failed: {exc}", + "reason": "schedule_failed", + "error": f"RAG sync scheduling failed: {exc}", } + return { + **_QUEUE_RECEIPT_FIELDS, + "success": True, + "scheduled": True, + } def maybe_delete_text_relation_doc_from_rag( diff --git a/src/backend/services/rag_text_relation_sync_service.py b/src/backend/services/rag_text_relation_sync_service.py index 57713ab08..2212a637e 100644 --- a/src/backend/services/rag_text_relation_sync_service.py +++ b/src/backend/services/rag_text_relation_sync_service.py @@ -2,22 +2,37 @@ from datetime import datetime from dataclasses import dataclass -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Tuple, +) from bson import ObjectId from bson.errors import InvalidId -from src.backend.db.mongo_client import get_concepts_collection +from src.backend.db.mongo_client import ( + get_scoped_knowledge_assertions_collection, +) from src.backend.db.repositories.text_value_repository import ( TextRelationsRepository, TextValuesRepository, ) from src.backend.security.access_control import ( - apply_concept_query_filter, + filter_accessible_concept_ids, override_current_organisation, override_current_user, ) from src.backend.services.rag_service import RAGBackendUnavailable, get_rag_service +from src.backend.services.text_relation_predicate_validation_service import ( + predicate_concept_id_for_storage, +) @dataclass(frozen=True) @@ -44,102 +59,43 @@ def list_text_relation_index_items( Note: `total` is the number of *visible* relations within `scan_limit`. """ - user_id, org_id = _parse_namespace(namespace) - rel_filter: Dict[str, Any] = {} - if predicates: - rel_filter["predicate"] = {"$in": list(predicates)} - safe_limit = max(int(limit), 0) safe_offset = max(int(offset), 0) safe_scan_limit = max(int(scan_limit), 0) - - relations = TextRelationsRepository.find( - rel_filter, - projection={ - "_id": 1, - "subject_concept_id": 1, - "predicate": 1, - "object_text_id": 1, - "updated_at": 1, - }, - sort=[("updated_at", -1)], + docs = collect_text_relation_docs_for_namespace( + namespace=namespace, + predicates=predicates, + languages=languages, limit=safe_scan_limit, ) - + total_visible = len(docs) + selected_docs = ( + docs[safe_offset : safe_offset + safe_limit] if safe_limit else [] + ) items: List[Dict[str, Any]] = [] - concept_visibility: Dict[str, bool] = {} - - total_visible = 0 - scanned = 0 - - for rel in relations: - scanned += 1 - if not isinstance(rel, dict): - continue - - relation_id = rel.get("_id") - relation_id_str = str(relation_id) if relation_id else None - subject_concept_id = rel.get("subject_concept_id") - predicate = rel.get("predicate") - object_text_id = rel.get("object_text_id") - - if ( - not relation_id_str - or not isinstance(subject_concept_id, str) - or not predicate - or not object_text_id - ): - continue - - if subject_concept_id not in concept_visibility: - concept_visibility[subject_concept_id] = _concept_visible_in_namespace( - concept_id=subject_concept_id, user_id=user_id, org_id=org_id - ) - if not concept_visibility[subject_concept_id]: - continue - - text_value_oid = _safe_object_id(object_text_id) - if text_value_oid is None: - continue - - tv = TextValuesRepository.find_one( - {"_id": text_value_oid}, {"text": 1, "lang": 1} - ) - if not isinstance(tv, dict): - continue - - text = tv.get("text") - lang = tv.get("lang") or "en-NZ" - if not isinstance(text, str) or not text.strip(): - continue - if languages and isinstance(lang, str) and lang not in set(languages): - continue - - visible_index = total_visible - total_visible += 1 - if safe_limit and visible_index < safe_offset: - continue - if safe_limit and len(items) >= safe_limit: - continue - + for doc in selected_docs: + metadata = doc.metadata items.append( { - "relation_id": relation_id_str, - "subject_concept_id": subject_concept_id, - "predicate": str(predicate), - "text_value_id": str(text_value_oid), - "lang": str(lang), - "preview_length": len(text), - "updated_at": ( - str(rel.get("updated_at")) if rel.get("updated_at") else None - ), + "index_item_id": doc.doc_id, + "row_kind": metadata.get("row_kind") or "base_text_relation", + "relation_id": metadata.get("relation_id"), + "assertion_id": metadata.get("assertion_id"), + "subject_concept_id": metadata.get("subject_concept_id"), + "predicate": metadata.get("predicate"), + "text_value_id": metadata.get("text_value_id"), + "lang": metadata.get("lang"), + "preview_length": metadata.get("content_length", len(doc.text)), + "updated_at": metadata.get("updated_at"), + "source": metadata.get("source"), } ) return { "items": items, "total": total_visible, - "scanned": scanned, + "scanned": total_visible, + "scanned_kind": "visible_projected_candidates", "limit": safe_limit, "offset": safe_offset, "scan_limit": safe_scan_limit, @@ -158,9 +114,55 @@ def get_text_relation_preview( if not isinstance(relation_id, str) or not relation_id.strip(): return None + requested_id = relation_id.strip() + scoped_assertion_id = ( + requested_id.removeprefix("scoped_assertion:") + if requested_id.startswith("scoped_assertion:") + else requested_id + if requested_id.startswith("ska_") + else None + ) + if scoped_assertion_id: + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + return None + assertion = collection.find_one( + { + "assertion_id": scoped_assertion_id, + "status": "asserted", + "object_kind": "text", + "scope.audience_keys": { + "$in": [f"user:{user_id}", f"org:{org_id}"], + }, + } + ) + if not isinstance(assertion, dict): + return None + concept_visibility: Dict[str, bool] = {} + _populate_concept_visibility( + [ + str(assertion.get("subject_concept_id") or ""), + str( + predicate_concept_id_for_storage( + assertion.get("predicate") + ) + or "" + ), + ], + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + return _scoped_assertion_to_rag_doc( + assertion, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + relation_key: Any - relation_oid = _safe_object_id(relation_id.strip()) - relation_key = relation_oid if relation_oid is not None else relation_id.strip() + relation_oid = _safe_object_id(requested_id) + relation_key = relation_oid if relation_oid is not None else requested_id rel = TextRelationsRepository.find_one( {"_id": relation_key}, @@ -184,9 +186,14 @@ def get_text_relation_preview( if not isinstance(subject_concept_id, str) or not predicate or not object_text_id: return None - if not _concept_visible_in_namespace( - concept_id=subject_concept_id, user_id=user_id, org_id=org_id - ): + predicate_concept_id = predicate_concept_id_for_storage(predicate) + if not predicate_concept_id: + return None + if _visible_concept_ids_in_namespace( + [subject_concept_id, predicate_concept_id], + user_id=user_id, + org_id=org_id, + ) != {subject_concept_id, predicate_concept_id}: return None text_value_oid = _safe_object_id(object_text_id) @@ -214,6 +221,7 @@ def get_text_relation_preview( "source": "vontology_text_relation", "subject_concept_id": subject_concept_id, "predicate": str(predicate), + "predicate_concept_id": predicate_concept_id, "relation_id": relation_id_str, "text_value_id": str(text_value_oid), "lang": str(lang), @@ -271,15 +279,92 @@ def _safe_object_id(value: Any) -> ObjectId | None: def _concept_visible_in_namespace( *, concept_id: str, user_id: str, org_id: str ) -> bool: - coll = get_concepts_collection() - if coll is None: - # Fail-open for dev tooling when DB is not available. - return True + return concept_id in _visible_concept_ids_in_namespace( + [concept_id], + user_id=user_id, + org_id=org_id, + ) + +def _visible_concept_ids_in_namespace( + concept_ids: Iterable[str], + *, + user_id: str, + org_id: str, +) -> set[str]: + """Resolve one bounded concept-visibility batch for a namespace.""" + + candidates = { + concept_id.strip() + for concept_id in concept_ids + if isinstance(concept_id, str) and concept_id.strip().startswith("#V#") + } + if not candidates: + return set() with override_current_user(user_id), override_current_organisation(org_id): - query = apply_concept_query_filter({"concept_id": concept_id}) - doc = coll.find_one(query, {"_id": 1}) - return bool(doc) + return filter_accessible_concept_ids(candidates) + + +def _populate_concept_visibility( + concept_ids: Iterable[str], + *, + user_id: str, + org_id: str, + concept_visibility: Dict[str, bool], +) -> None: + """Populate an explicit cache with one query for all missing concept IDs.""" + + missing = { + concept_id + for concept_id in concept_ids + if isinstance(concept_id, str) + and concept_id.startswith("#V#") + and concept_id not in concept_visibility + } + if not missing: + return + visible = _visible_concept_ids_in_namespace( + missing, + user_id=user_id, + org_id=org_id, + ) + for concept_id in missing: + concept_visibility[concept_id] = concept_id in visible + + +def _populate_text_value_cache( + relations: Iterable[Any], + *, + text_value_cache: Dict[str, Mapping[str, Any] | None], +) -> None: + """Hydrate one bounded relation batch with one TextValue query.""" + + object_ids: dict[str, ObjectId] = {} + for relation in relations: + if not isinstance(relation, Mapping): + continue + object_id = _safe_object_id(relation.get("object_text_id")) + if object_id is None: + continue + cache_key = str(object_id) + if cache_key not in text_value_cache: + object_ids[cache_key] = object_id + if not object_ids: + return + for cache_key in object_ids: + text_value_cache[cache_key] = None + for text_value in TextValuesRepository.find( + {"_id": {"$in": list(object_ids.values())}}, + { + "text": 1, + "lang": 1, + }, + ): + if not isinstance(text_value, Mapping): + continue + cache_key = str(text_value.get("_id") or "") + if cache_key in text_value_cache: + text_value_cache[cache_key] = text_value def _build_rag_text(*, concept_id: str, predicate: str, lang: str, text: str) -> str: @@ -292,6 +377,582 @@ def _build_rag_text(*, concept_id: str, predicate: str, lang: str, text: str) -> ) +def _base_relation_to_rag_doc( + relation: Any, + *, + user_id: str, + org_id: str, + concept_allow: set[str] | None, + language_allow: set[str], + concept_visibility: Dict[str, bool], + text_value_cache: Dict[str, Mapping[str, Any] | None] | None = None, +) -> TextRelationRagDoc | None: + if not isinstance(relation, dict): + return None + + relation_id = relation.get("_id") + relation_id_str = str(relation_id) if relation_id else None + subject_concept_id = relation.get("subject_concept_id") + predicate = relation.get("predicate") + object_text_id = relation.get("object_text_id") + if ( + not relation_id_str + or not isinstance(subject_concept_id, str) + or not predicate + ): + return None + if concept_allow is not None and subject_concept_id not in concept_allow: + return None + + predicate_concept_id = predicate_concept_id_for_storage(predicate) + if not predicate_concept_id: + return None + for concept_id in (subject_concept_id, predicate_concept_id): + if concept_id not in concept_visibility: + concept_visibility[concept_id] = _concept_visible_in_namespace( + concept_id=concept_id, + user_id=user_id, + org_id=org_id, + ) + if not concept_visibility[concept_id]: + return None + + text_value_oid = _safe_object_id(object_text_id) + if text_value_oid is None: + return None + cache_key = str(text_value_oid) + text_value = ( + text_value_cache.get(cache_key) + if text_value_cache is not None and cache_key in text_value_cache + else TextValuesRepository.find_one({"_id": text_value_oid}) + ) + if not isinstance(text_value, Mapping): + return None + + text = text_value.get("text") + lang = str(text_value.get("lang") or "en-NZ") + if not isinstance(text, str) or not text.strip(): + return None + if language_allow and lang not in language_allow: + return None + + return TextRelationRagDoc( + doc_id=f"text_relation:{relation_id_str}", + text=_build_rag_text( + concept_id=subject_concept_id, + predicate=str(predicate), + lang=lang, + text=text, + ), + metadata={ + "type": "text_relation", + "source": "vontology_text_relation", + "concept_id": subject_concept_id, + "subject_concept_id": subject_concept_id, + "predicate": str(predicate), + "predicate_concept_id": predicate_concept_id, + "relation_id": relation_id_str, + "text_value_id": str(text_value_oid), + "lang": lang, + "language": lang, + "content_length": len(text), + "created_at": ( + str(relation.get("created_at")) + if relation.get("created_at") + else None + ), + "updated_at": ( + str(relation.get("updated_at")) + if relation.get("updated_at") + else None + ), + "user_id": user_id, + "organisation_concept_id": org_id, + "org_id": org_id, + }, + ) + + +def _scoped_assertion_to_rag_doc( + assertion: Any, + *, + user_id: str, + org_id: str, + concept_visibility: Dict[str, bool], +) -> TextRelationRagDoc | None: + if not isinstance(assertion, dict): + return None + object_text = assertion.get("object_text") + if not isinstance(object_text, dict): + return None + + text = object_text.get("text") + lang = str(object_text.get("language") or "en-NZ") + subject_concept_id = str(assertion.get("subject_concept_id") or "") + predicate = str(assertion.get("predicate") or "") + assertion_id = str(assertion.get("assertion_id") or "") + if not text or not subject_concept_id or not predicate or not assertion_id: + return None + predicate_concept_id = predicate_concept_id_for_storage(predicate) + if not predicate_concept_id: + return None + + for concept_id in (subject_concept_id, predicate_concept_id): + if concept_id not in concept_visibility: + concept_visibility[concept_id] = _concept_visible_in_namespace( + concept_id=concept_id, + user_id=user_id, + org_id=org_id, + ) + if not concept_visibility[concept_id]: + return None + + scope = assertion.get("scope") + scope = scope if isinstance(scope, dict) else {} + provenance = assertion.get("provenance") + provenance = provenance if isinstance(provenance, dict) else {} + + return TextRelationRagDoc( + doc_id=f"scoped_assertion:{assertion_id}", + text=_build_rag_text( + concept_id=subject_concept_id, + predicate=predicate, + lang=lang, + text=str(text), + ), + metadata={ + "type": "scoped_knowledge_assertion", + "source": "scoped_knowledge_assertion", + "concept_id": subject_concept_id, + "subject_concept_id": subject_concept_id, + "predicate": predicate, + "predicate_concept_id": predicate_concept_id, + "assertion_id": assertion_id, + "relation_id": None, + "row_kind": "scoped_assertion", + "lang": lang, + "language": lang, + "content_length": len(str(text)), + "user_id": user_id, + "organisation_concept_id": org_id, + "org_id": org_id, + "audience_user_concept_id": user_id, + "audience_organisation_concept_id": org_id, + "audience_keys": list(scope.get("audience_keys") or []), + "assertion_scope": scope, + "assertion_provenance": provenance, + "asserted_by_user_concept_id": provenance.get( + "asserted_by_user_concept_id" + ), + "asserted_by_organisation_concept_id": provenance.get( + "organisation_concept_id" + ), + "canonical_publication": False, + "created_at": ( + str(assertion.get("created_at")) + if assertion.get("created_at") + else None + ), + "updated_at": ( + str(assertion.get("updated_at")) + if assertion.get("updated_at") + else None + ), + }, + ) + + +def _iter_bounded_batches( + items: Iterable[Any], + *, + batch_size: int, +) -> Iterable[List[Any]]: + """Yield bounded in-memory batches from one already-open source cursor.""" + + batch: List[Any] = [] + for item in items: + batch.append(item) + if len(batch) >= batch_size: + yield batch + batch = [] + if batch: + yield batch + + +def _emit_stale_doc_batch( + stale_batch: Sequence[str], + *, + stale_doc_ids: List[str] | None, + stale_doc_sink: Callable[[Sequence[str]], None] | None, +) -> None: + """Publish one bounded stale-document batch to compatibility consumers.""" + + if not stale_batch: + return + deduplicated = list(dict.fromkeys(stale_batch)) + if stale_doc_ids is not None: + stale_doc_ids.extend(deduplicated) + if stale_doc_sink is not None: + stale_doc_sink(deduplicated) + + +def _base_relation_should_prune( + relation: Any, + *, + concept_allow: set[str] | None, + language_allow: set[str], + concept_visibility: Mapping[str, bool], + text_value_cache: Mapping[str, Mapping[str, Any] | None], +) -> bool: + """Return whether a scanned base relation is now ineligible for this index.""" + + if not isinstance(relation, Mapping) or not relation.get("_id"): + return False + subject_id = str(relation.get("subject_concept_id") or "") + if concept_allow is not None and subject_id not in concept_allow: + return False + predicate_id = predicate_concept_id_for_storage(relation.get("predicate")) + if ( + not subject_id + or not predicate_id + or not concept_visibility.get(subject_id, False) + or not concept_visibility.get(predicate_id, False) + ): + return True + text_value_id = _safe_object_id(relation.get("object_text_id")) + if text_value_id is None: + return True + text_value = text_value_cache.get(str(text_value_id)) + if not isinstance(text_value, Mapping): + return True + lang = str(text_value.get("lang") or "en-NZ") + if language_allow and lang not in language_allow: + return False + text = text_value.get("text") + return not isinstance(text, str) or not text.strip() + + +def iter_text_relation_docs_for_namespace( + *, + namespace: str, + predicates: Optional[Sequence[str]] = None, + languages: Optional[Sequence[str]] = None, + concept_ids: Optional[Sequence[str]] = None, + updated_since: Optional[datetime] = None, + limit: int | None = None, + source_batch_size: int = 1000, + stale_doc_ids: List[str] | None = None, + stale_doc_sink: Callable[[Sequence[str]], None] | None = None, +) -> Iterable[TextRelationRagDoc]: + """Stream one stable base-then-scoped namespace scan. + + Each backing store is opened once and traversed in immutable ``_id`` order. + Bounded in-memory batches provide batched visibility checks without + repeatedly rescanning from zero or paging on mutable visible offsets. + The scan is deliberately limited to the one supplied namespace; it does not + fan out to other users or organisations. + """ + + user_id, org_id = _parse_namespace(namespace) + safe_limit = None if limit is None else max(int(limit), 0) + if safe_limit == 0: + return + safe_batch_size = max(1, min(int(source_batch_size), 5000)) + + rel_filter: Dict[str, Any] = {} + concept_allow: Optional[set[str]] = None + if concept_ids: + concept_allow = { + concept_id.strip() + for concept_id in concept_ids + if isinstance(concept_id, str) and concept_id.strip() + } + if concept_allow: + rel_filter["subject_concept_id"] = {"$in": sorted(concept_allow)} + if predicates: + rel_filter["predicate"] = {"$in": list(predicates)} + if updated_since is not None: + rel_filter["updated_at"] = {"$gte": updated_since} + + language_allow = set(languages or ()) + prune_stale = stale_doc_ids is not None or stale_doc_sink is not None + yielded = 0 + base_sort = ( + [("updated_at", 1), ("_id", 1)] + if updated_since is not None and not concept_allow and not predicates + else [("_id", 1)] + ) + base_cursor = TextRelationsRepository.find( + rel_filter, + sort=base_sort, + ) + set_batch_size = getattr(base_cursor, "batch_size", None) + if callable(set_batch_size): + base_cursor = set_batch_size(safe_batch_size) + for relations in _iter_bounded_batches( + base_cursor, + batch_size=safe_batch_size, + ): + concept_visibility: Dict[str, bool] = {} + text_value_cache: Dict[str, Mapping[str, Any] | None] = {} + _populate_concept_visibility( + ( + concept_id + for relation in relations + if isinstance(relation, Mapping) + for concept_id in ( + str(relation.get("subject_concept_id") or ""), + str( + predicate_concept_id_for_storage( + relation.get("predicate") + ) + or "" + ), + ) + ), + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + _populate_text_value_cache( + relations, + text_value_cache=text_value_cache, + ) + batch_docs: List[TextRelationRagDoc] = [] + stale_batch: List[str] = [] + remaining = ( + None if safe_limit is None else max(safe_limit - yielded, 0) + ) + for relation in relations: + doc = _base_relation_to_rag_doc( + relation, + user_id=user_id, + org_id=org_id, + concept_allow=concept_allow, + language_allow=language_allow, + concept_visibility=concept_visibility, + text_value_cache=text_value_cache, + ) + if doc is None: + if ( + prune_stale + and _base_relation_should_prune( + relation, + concept_allow=concept_allow, + language_allow=language_allow, + concept_visibility=concept_visibility, + text_value_cache=text_value_cache, + ) + ): + stale_batch.append( + f"text_relation:{relation.get('_id')}" + ) + continue + batch_docs.append(doc) + if remaining is not None and len(batch_docs) >= remaining: + break + _emit_stale_doc_batch( + stale_batch, + stale_doc_ids=stale_doc_ids, + stale_doc_sink=stale_doc_sink, + ) + for doc in batch_docs: + yield doc + yielded += 1 + if safe_limit is not None and yielded >= safe_limit: + return + + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + return + scoped_query: Dict[str, Any] = { + "scope.audience_keys": { + "$in": [f"user:{user_id}", f"org:{org_id}"], + }, + "status": ( + {"$in": ["asserted", "retracted"]} + if prune_stale + else "asserted" + ), + "object_kind": "text", + } + if updated_since is not None: + scoped_query["updated_at"] = {"$gte": updated_since} + if concept_allow: + scoped_query["subject_concept_id"] = {"$in": sorted(concept_allow)} + if predicates: + scoped_query["predicate"] = {"$in": list(predicates)} + if languages: + scoped_query["object_text.language"] = {"$in": list(languages)} + + scoped_cursor = collection.find(scoped_query).sort([("_id", 1)]) + set_batch_size = getattr(scoped_cursor, "batch_size", None) + if callable(set_batch_size): + scoped_cursor = set_batch_size(safe_batch_size) + for assertions in _iter_bounded_batches( + scoped_cursor, + batch_size=safe_batch_size, + ): + concept_visibility: Dict[str, bool] = {} + batch_concept_ids: set[str] = set() + for assertion in assertions: + if not isinstance(assertion, dict): + continue + subject_id = str(assertion.get("subject_concept_id") or "") + predicate_id = predicate_concept_id_for_storage( + assertion.get("predicate") + ) + if subject_id: + batch_concept_ids.add(subject_id) + if predicate_id: + batch_concept_ids.add(predicate_id) + _populate_concept_visibility( + batch_concept_ids, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + batch_docs = [] + stale_batch = [] + remaining = ( + None if safe_limit is None else max(safe_limit - yielded, 0) + ) + for assertion in assertions: + if ( + prune_stale + and isinstance(assertion, Mapping) + and assertion.get("status") != "asserted" + ): + assertion_id = str(assertion.get("assertion_id") or "").strip() + if assertion_id: + stale_batch.append( + f"scoped_assertion:{assertion_id}" + ) + continue + doc = _scoped_assertion_to_rag_doc( + assertion, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + if doc is None: + if prune_stale and isinstance( + assertion, + Mapping, + ): + assertion_id = str( + assertion.get("assertion_id") or "" + ).strip() + if assertion_id: + stale_batch.append( + f"scoped_assertion:{assertion_id}" + ) + continue + batch_docs.append(doc) + if remaining is not None and len(batch_docs) >= remaining: + break + _emit_stale_doc_batch( + stale_batch, + stale_doc_ids=stale_doc_ids, + stale_doc_sink=stale_doc_sink, + ) + for doc in batch_docs: + yield doc + yielded += 1 + if safe_limit is not None and yielded >= safe_limit: + return + + +def _collect_scoped_text_assertion_docs( + *, + user_id: str, + org_id: str, + predicates: Optional[Sequence[str]], + languages: Optional[Sequence[str]], + concept_allow: set[str] | None, + updated_since: datetime | None, + skip: int, + limit: int, +) -> List[TextRelationRagDoc]: + """Collect one stable visible page while advancing through raw candidates. + + The public offset is in projected, currently visible rows. Raw Mongo + offsets are advanced independently so revoked subjects or predicates do + not shorten a non-terminal page or cause the next visible page to repeat. + """ + + if limit <= 0: + return [] + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + return [] + + query: Dict[str, Any] = { + "scope.audience_keys": { + "$in": [f"user:{user_id}", f"org:{org_id}"], + }, + "status": "asserted", + "object_kind": "text", + } + if updated_since is not None: + query["updated_at"] = {"$gte": updated_since} + if concept_allow: + query["subject_concept_id"] = {"$in": sorted(concept_allow)} + if predicates: + query["predicate"] = {"$in": list(predicates)} + if languages: + query["object_text.language"] = {"$in": list(languages)} + + visible_skip = max(int(skip), 0) + target_end = visible_skip + max(int(limit), 0) + raw_batch_size = max(1, min(target_end, 1000)) + raw_offset = 0 + raw_exhausted = False + visible_docs: List[TextRelationRagDoc] = [] + while len(visible_docs) < target_end and not raw_exhausted: + raw_candidates = list( + collection.find(query) + .sort([("updated_at", -1), ("assertion_id", 1)]) + .skip(raw_offset) + .limit(raw_batch_size) + ) + raw_offset += len(raw_candidates) + raw_exhausted = len(raw_candidates) < raw_batch_size + if not raw_candidates: + break + concept_visibility: Dict[str, bool] = {} + batch_concept_ids: set[str] = set() + for assertion in raw_candidates: + if not isinstance(assertion, dict): + continue + subject_id = str(assertion.get("subject_concept_id") or "") + predicate_id = predicate_concept_id_for_storage( + assertion.get("predicate") + ) + if subject_id: + batch_concept_ids.add(subject_id) + if predicate_id: + batch_concept_ids.add(predicate_id) + _populate_concept_visibility( + batch_concept_ids, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + for assertion in raw_candidates: + doc = _scoped_assertion_to_rag_doc( + assertion, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + if doc is not None: + visible_docs.append(doc) + if len(visible_docs) >= target_end: + break + return visible_docs[visible_skip:target_end] + + def collect_text_relation_docs_for_namespace( *, namespace: str, @@ -329,158 +990,90 @@ def collect_text_relation_docs_for_namespace( safe_limit = max(int(limit), 0) safe_sort = sort if sort is not None else [("_id", 1)] - relations = list( - TextRelationsRepository.find( - rel_filter, - sort=safe_sort, - skip=safe_skip, - limit=safe_limit, - ) - ) - docs: List[TextRelationRagDoc] = [] - - # Small cache to avoid repeated concept lookups. - concept_visibility: Dict[str, bool] = {} - - for rel in relations: - if not isinstance(rel, dict): - continue - - relation_id = rel.get("_id") - relation_id_str = str(relation_id) if relation_id else None - subject_concept_id = rel.get("subject_concept_id") - predicate = rel.get("predicate") - object_text_id = rel.get("object_text_id") - - if ( - not relation_id_str - or not isinstance(subject_concept_id, str) - or not predicate - ): - continue - - if concept_allow is not None and subject_concept_id not in concept_allow: - continue + if safe_limit == 0: + return [] + + # Combined paging is base-publication rows followed by actor-visible scoped + # rows. Scan base rows until the requested page is full or base storage is + # exhausted; only then map the remaining offset/budget into scoped storage. + target_end = safe_skip + safe_limit + scan_batch_size = max(1, min(target_end, 1000)) + raw_base_skip = 0 + base_exhausted = False + base_docs: List[TextRelationRagDoc] = [] + language_allow = set(languages or ()) - if subject_concept_id not in concept_visibility: - concept_visibility[subject_concept_id] = _concept_visible_in_namespace( - concept_id=subject_concept_id, user_id=user_id, org_id=org_id + while len(base_docs) < target_end and not base_exhausted: + relations = list( + TextRelationsRepository.find( + rel_filter, + sort=safe_sort, + skip=raw_base_skip, + limit=scan_batch_size, ) - if not concept_visibility[subject_concept_id]: - continue - - text_value_oid = _safe_object_id(object_text_id) - if text_value_oid is None: - continue - - tv = TextValuesRepository.find_one({"_id": text_value_oid}) - if not isinstance(tv, dict): - continue - - text = tv.get("text") - lang = tv.get("lang") or "en-NZ" - if not isinstance(text, str) or not text.strip(): - continue - if languages and isinstance(lang, str) and lang not in set(languages): - continue - - rag_text = _build_rag_text( - concept_id=subject_concept_id, - predicate=str(predicate), - lang=str(lang), - text=text, ) - - metadata = { - # Distinguish from chat history docs. - "type": "text_relation", - "source": "vontology_text_relation", - # Preferred canonical key for concept. - "concept_id": subject_concept_id, - "subject_concept_id": subject_concept_id, - "predicate": str(predicate), - "relation_id": relation_id_str, - "text_value_id": str(text_value_oid), - "lang": str(lang), - "language": str(lang), - # Required for query-time filtering. - "user_id": user_id, - "organisation_concept_id": org_id, - # Alias to match Jira ticket naming. - "org_id": org_id, - } - - docs.append( - TextRelationRagDoc( - doc_id=f"text_relation:{relation_id_str}", - text=rag_text, - metadata=metadata, - ) + raw_base_skip += len(relations) + base_exhausted = len(relations) < scan_batch_size + if not relations: + break + concept_visibility: Dict[str, bool] = {} + text_value_cache: Dict[str, Mapping[str, Any] | None] = {} + _populate_concept_visibility( + ( + concept_id + for relation in relations + if isinstance(relation, Mapping) + for concept_id in ( + str(relation.get("subject_concept_id") or ""), + str( + predicate_concept_id_for_storage( + relation.get("predicate") + ) + or "" + ), + ) + ), + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, ) - - from src.backend.services.scoped_assertion_service import ( - list_visible_scoped_assertions, - ) - - scoped_assertions = list_visible_scoped_assertions( - subject_concept_ids=sorted(concept_allow) if concept_allow else None, + _populate_text_value_cache( + relations, + text_value_cache=text_value_cache, + ) + for relation in relations: + doc = _base_relation_to_rag_doc( + relation, + user_id=user_id, + org_id=org_id, + concept_allow=concept_allow, + language_allow=language_allow, + concept_visibility=concept_visibility, + text_value_cache=text_value_cache, + ) + if doc is not None: + base_docs.append(doc) + if len(base_docs) >= target_end: + break + + page = base_docs[safe_skip:target_end] + remaining = safe_limit - len(page) + if remaining <= 0 or not base_exhausted: + return page + + scoped_skip = max(safe_skip - len(base_docs), 0) + scoped_docs = _collect_scoped_text_assertion_docs( + user_id=user_id, + org_id=org_id, predicates=predicates, - object_kind="text", - limit=max(safe_limit, 1), - user_concept_id=user_id, - organisation_concept_id=org_id, + languages=languages, + concept_allow=concept_allow, + updated_since=updated_since, + skip=scoped_skip, + limit=remaining, ) - language_allow = set(languages or ()) - for assertion in scoped_assertions: - object_text = assertion.get("object_text") - if not isinstance(object_text, dict): - continue - text = object_text.get("text") - lang = str(object_text.get("language") or "en-NZ") - subject_concept_id = str(assertion.get("subject_concept_id") or "") - predicate = str(assertion.get("predicate") or "") - assertion_id = str(assertion.get("assertion_id") or "") - if not text or not subject_concept_id or not predicate or not assertion_id: - continue - if language_allow and lang not in language_allow: - continue - if updated_since is not None: - updated_at = assertion.get("updated_at") - if isinstance(updated_at, str): - try: - if datetime.fromisoformat(updated_at) < updated_since: - continue - except ValueError: - continue - docs.append( - TextRelationRagDoc( - doc_id=f"scoped_assertion:{assertion_id}", - text=_build_rag_text( - concept_id=subject_concept_id, - predicate=predicate, - lang=lang, - text=str(text), - ), - metadata={ - "type": "scoped_knowledge_assertion", - "source": "scoped_knowledge_assertion", - "concept_id": subject_concept_id, - "subject_concept_id": subject_concept_id, - "predicate": predicate, - "assertion_id": assertion_id, - "relation_id": assertion_id, - "lang": lang, - "language": lang, - "user_id": user_id, - "organisation_concept_id": org_id, - "org_id": org_id, - "assertion_scope": assertion.get("scope"), - "canonical_publication": False, - }, - ) - ) - - return docs + page.extend(scoped_docs[:remaining]) + return page def sync_text_relations_to_rag( @@ -502,16 +1095,43 @@ def sync_text_relations_to_rag( except RAGBackendUnavailable as exc: return {"success": False, "error": f"RAG service unavailable: {exc}"} - docs = collect_text_relation_docs_for_namespace( - namespace=namespace, - predicates=predicates, - languages=languages, - concept_ids=concept_ids, - updated_since=updated_since, - skip=skip, - sort=sort, - limit=limit, - ) + delete_batch_size = max(int(batch_size), 1) + deleted = 0 + + def _delete_stale_batch(stale_ids: Sequence[str]) -> None: + nonlocal deleted + deduplicated = list(dict.fromkeys(stale_ids)) + for start in range(0, len(deduplicated), delete_batch_size): + deleted += int( + rag.delete_documents( + deduplicated[start : start + delete_batch_size], + namespace=namespace, + ) + ) + + if skip == 0 and sort in (None, [("_id", 1)]): + docs = list( + iter_text_relation_docs_for_namespace( + namespace=namespace, + predicates=predicates, + languages=languages, + concept_ids=concept_ids, + updated_since=updated_since, + limit=limit, + stale_doc_sink=_delete_stale_batch, + ) + ) + else: + docs = collect_text_relation_docs_for_namespace( + namespace=namespace, + predicates=predicates, + languages=languages, + concept_ids=concept_ids, + updated_since=updated_since, + skip=skip, + sort=sort, + limit=limit, + ) added = 0 failed = 0 @@ -537,6 +1157,109 @@ def _chunks( "total_candidates": len(docs), "added": added, "failed": failed, + **({"deleted": int(deleted)} if deleted else {}), + } + + +def sync_scoped_assertions_to_rag( + *, + namespace: str, + assertion_ids: Sequence[str], +) -> Dict[str, Any]: + """Refresh exact scoped assertions without a base-relation scan budget.""" + + user_id, org_id = _parse_namespace(namespace) + requested_ids = sorted( + { + str(assertion_id).strip() + for assertion_id in assertion_ids + if isinstance(assertion_id, str) + and str(assertion_id).strip().startswith("ska_") + } + )[:100] + if not requested_ids: + return { + "success": False, + "namespace": namespace, + "error": "assertion_ids_required", + } + try: + rag = get_rag_service() + except RAGBackendUnavailable as exc: + return {"success": False, "error": f"RAG service unavailable: {exc}"} + + collection = get_scoped_knowledge_assertions_collection() + docs: List[TextRelationRagDoc] = [] + if collection is not None: + assertions = list( + collection.find( + { + "assertion_id": {"$in": requested_ids}, + "status": "asserted", + "object_kind": "text", + "scope.audience_keys": { + "$in": [f"user:{user_id}", f"org:{org_id}"], + }, + } + ) + ) + concept_visibility: Dict[str, bool] = {} + assertion_concept_ids: set[str] = set() + for assertion in assertions: + if not isinstance(assertion, dict): + continue + subject_id = str(assertion.get("subject_concept_id") or "") + predicate_id = predicate_concept_id_for_storage( + assertion.get("predicate") + ) + if subject_id: + assertion_concept_ids.add(subject_id) + if predicate_id: + assertion_concept_ids.add(predicate_id) + _populate_concept_visibility( + assertion_concept_ids, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + for assertion in assertions: + doc = _scoped_assertion_to_rag_doc( + assertion, + user_id=user_id, + org_id=org_id, + concept_visibility=concept_visibility, + ) + if doc is not None: + docs.append(doc) + + payload = [ + {"id": doc.doc_id, "text": doc.text, "metadata": doc.metadata} + for doc in docs + ] + added = 0 + failed = 0 + if payload: + ok, bad = rag.upsert_documents(payload, namespace=namespace) + added = int(ok) + failed = int(bad) + + indexed_ids = { + str(doc.metadata.get("assertion_id") or "").strip() for doc in docs + } + stale_ids = [ + f"scoped_assertion:{assertion_id}" + for assertion_id in requested_ids + if assertion_id not in indexed_ids + ] + deleted = rag.delete_documents(stale_ids, namespace=namespace) if stale_ids else 0 + return { + "success": failed == 0, + "namespace": namespace, + "requested": len(requested_ids), + "added": added, + "failed": failed, + "deleted": int(deleted), + "sync_scope": "exact_scoped_assertions", } diff --git a/src/backend/services/scoped_assertion_service.py b/src/backend/services/scoped_assertion_service.py index a797db1a6..840fb2a27 100644 --- a/src/backend/services/scoped_assertion_service.py +++ b/src/backend/services/scoped_assertion_service.py @@ -14,16 +14,20 @@ from typing import Any, Mapping, Sequence from pymongo import ReturnDocument +from pymongo.errors import DuplicateKeyError from ..db.mongo_client import get_scoped_knowledge_assertions_collection from ..security.access_control import ( can_access_concept, + filter_accessible_concept_ids, get_effective_organisation_concept_id, get_effective_user_concept_id, + override_current_actor, ) from .namespace_service import resolve_canonical_namespace from .relationship_write_service import validate_predicate_concept from .text_relation_predicate_validation_service import ( + predicate_concept_id_for_storage, resolve_text_relation_predicate_for_write, ) @@ -31,6 +35,10 @@ STORAGE_SURFACE = "scoped_knowledge_assertions" SCOPE_MODES = frozenset({"user", "organisation"}) OBJECT_KINDS = frozenset({"text", "concept"}) +MAX_PAGE_LIMIT = 1000 +MAX_PAGE_SUBJECTS = 100 +MAX_PAGE_CANDIDATES = 10_000 +MAX_RETRACTION_HISTORY = 20 def _clean_concept_id(value: Any, field_name: str) -> str: @@ -58,26 +66,44 @@ def _normalise_evidence(value: Any) -> dict[str, Any]: raise ValueError("evidence must be JSON-serialisable") from exc -def _scope_for_actor( +def _resolve_actor_context( *, - scope_mode: str, acting_user_concept_id: Any, organisation_concept_id: Any, namespace: Any, ) -> dict[str, Any]: + """Resolve explicit trusted actor values without enlarging ambient scope.""" + user_id = _clean_concept_id( acting_user_concept_id, "acting_user_concept_id", ) - org_id = _clean_optional_concept_id(organisation_concept_id) - mode = str(scope_mode or "user").strip().lower() - if mode not in SCOPE_MODES: - raise ValueError("scope_mode must be 'user' or 'organisation'") - if mode == "organisation" and not org_id: - raise ValueError( - "organisation scope requires a trusted organisation concept ID" + supplied_org_id = ( + _clean_concept_id( + organisation_concept_id, + "organisation_concept_id", ) - + if str(organisation_concept_id or "").strip() + else None + ) + ambient_user_id = _clean_optional_concept_id(get_effective_user_concept_id()) + ambient_org_id = _clean_optional_concept_id( + get_effective_organisation_concept_id() + ) + has_ambient_actor = bool(ambient_user_id or ambient_org_id) + if has_ambient_actor and user_id != ambient_user_id: + raise PermissionError( + "acting user does not match the current trusted actor" + ) + if ( + supplied_org_id + and has_ambient_actor + and supplied_org_id != ambient_org_id + ): + raise PermissionError( + "organisation does not match the current trusted actor" + ) + org_id = supplied_org_id or ambient_org_id actor_namespace = resolve_canonical_namespace( None, user_id, @@ -90,14 +116,54 @@ def _scope_for_actor( raise ValueError( "namespace does not match the trusted user and organisation scope" ) - audience_key = f"user:{user_id}" if mode == "user" else f"org:{org_id}" return { - "mode": mode, "user_concept_id": user_id, "organisation_concept_id": org_id, "namespace": actor_namespace, + } + + +def _scope_for_actor( + *, + scope_mode: str, + acting_user_concept_id: Any, + organisation_concept_id: Any, + namespace: Any, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Resolve assertion scope and the trusted actor context that created it.""" + + actor_context = _resolve_actor_context( + acting_user_concept_id=acting_user_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + ) + user_id = actor_context["user_concept_id"] + org_id = actor_context["organisation_concept_id"] + mode = str(scope_mode or "user").strip().lower() + if mode not in SCOPE_MODES: + raise ValueError("scope_mode must be 'user' or 'organisation'") + if mode == "organisation" and not org_id: + raise ValueError( + "organisation scope requires a trusted organisation concept ID" + ) + + audience_key = f"user:{user_id}" if mode == "user" else f"org:{org_id}" + assertion_org_id = org_id if mode == "organisation" else None + assertion_namespace = resolve_canonical_namespace( + None, + user_id, + assertion_org_id, + ) + scope = { + "mode": mode, + "user_concept_id": user_id, + # User-scoped knowledge follows the user independently of whichever + # organisation supplied the turn context that first established it. + "organisation_concept_id": assertion_org_id, + "namespace": assertion_namespace, "audience_keys": [audience_key], } + return scope, actor_context def _assertion_identity(identity: Mapping[str, Any]) -> str: @@ -105,15 +171,43 @@ def _assertion_identity(identity: Mapping[str, Any]) -> str: return f"ska_{hashlib.sha256(encoded.encode('utf-8')).hexdigest()[:32]}" +def _json_safe_value(value: Any, *, path: str) -> Any: + """Recursively convert supported BSON values without hiding bad data.""" + + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, Mapping): + result: dict[str, Any] = {} + for key, nested_value in value.items(): + if not isinstance(key, str): + raise TypeError(f"{path} contains a non-string object key") + result[key] = _json_safe_value( + nested_value, + path=f"{path}.{key}", + ) + return result + if isinstance(value, (list, tuple)): + return [ + _json_safe_value(item, path=f"{path}[{index}]") + for index, item in enumerate(value) + ] + raise TypeError( + f"{path} contains unsupported value type {type(value).__name__}" + ) + + def _serialise(document: Mapping[str, Any] | None) -> dict[str, Any] | None: if not isinstance(document, Mapping): return None - result = {key: value for key, value in document.items() if key != "_id"} - for field in ("created_at", "updated_at"): - value = result.get(field) - if isinstance(value, datetime): - result[field] = value.isoformat() - return result + serialised = _json_safe_value( + {key: value for key, value in document.items() if key != "_id"}, + path="assertion", + ) + if not isinstance(serialised, dict): + raise TypeError("assertion serialisation did not produce an object") + return serialised def upsert_scoped_assertion( @@ -135,55 +229,93 @@ def upsert_scoped_assertion( if canonical_publication is not False: raise ValueError("scoped assertions cannot request canonical publication") - subject_id = _clean_concept_id(subject_concept_id, "subject_concept_id") - if not can_access_concept(subject_id): - raise PermissionError("subject concept is not visible to the current actor") - - has_text = isinstance(target_text, str) and bool(target_text.strip()) - has_concept = isinstance(target_concept_id, str) and bool(target_concept_id.strip()) - if has_text == has_concept: - raise ValueError("provide exactly one of target_text or target_concept_id") - - if has_text: - resolution = resolve_text_relation_predicate_for_write(predicate) - storage_predicate = resolution.storage_predicate - object_kind = "text" - object_text = str(target_text).strip() - language_token = str(language or "en-NZ").strip() or "en-NZ" - object_concept_id = None - identity_object: dict[str, Any] = { - "text": object_text, - "language": language_token, - } - else: - storage_predicate = _clean_concept_id(predicate, "predicate") - is_valid, error_code, error_details = validate_predicate_concept( - storage_predicate - ) - if not is_valid: - raise ValueError( - f"{error_code or 'invalid_predicate'}: {error_details or {}}" - ) - object_kind = "concept" - object_text = None - language_token = None - object_concept_id = _clean_concept_id( - target_concept_id, - "target_concept_id", - ) - if not can_access_concept(object_concept_id): - raise PermissionError("target concept is not visible to the current actor") - identity_object = {"concept_id": object_concept_id} - - scope = _scope_for_actor( + scope, actor_context = _scope_for_actor( scope_mode=scope_mode, acting_user_concept_id=acting_user_concept_id, organisation_concept_id=organisation_concept_id, namespace=namespace, ) + with override_current_actor( + actor_context["user_concept_id"], + actor_context["organisation_concept_id"], + ): + subject_id = _clean_concept_id( + subject_concept_id, + "subject_concept_id", + ) + if not can_access_concept(subject_id): + raise PermissionError( + "subject concept is not visible to the current actor" + ) + + has_text = isinstance(target_text, str) and bool(target_text.strip()) + has_concept = isinstance(target_concept_id, str) and bool( + target_concept_id.strip() + ) + if has_text == has_concept: + raise ValueError( + "provide exactly one of target_text or target_concept_id" + ) + + if has_text: + requested_predicate_id = predicate_concept_id_for_storage( + predicate + ) + if requested_predicate_id and not can_access_concept( + requested_predicate_id + ): + raise PermissionError( + "predicate concept is not visible to the current actor" + ) + resolution = resolve_text_relation_predicate_for_write(predicate) + storage_predicate = resolution.storage_predicate + if not can_access_concept(resolution.predicate_concept_id): + raise PermissionError( + "predicate concept is not visible to the current actor" + ) + object_kind = "text" + object_text = str(target_text).strip() + language_token = str(language or "en-NZ").strip() or "en-NZ" + object_concept_id = None + identity_object: dict[str, Any] = { + "text": object_text, + "language": language_token, + } + else: + storage_predicate = _clean_concept_id(predicate, "predicate") + # Visibility is an authority boundary. Do not let predicate + # validation reveal or inspect a concept the actor cannot see. + if not can_access_concept(storage_predicate): + raise PermissionError( + "predicate concept is not visible to the current actor" + ) + is_valid, error_code, error_details = validate_predicate_concept( + storage_predicate + ) + if not is_valid: + raise ValueError( + f"{error_code or 'invalid_predicate'}: {error_details or {}}" + ) + object_kind = "concept" + object_text = None + language_token = None + object_concept_id = _clean_concept_id( + target_concept_id, + "target_concept_id", + ) + if not can_access_concept(object_concept_id): + raise PermissionError( + "target concept is not visible to the current actor" + ) + identity_object = {"concept_id": object_concept_id} + identity = { "scope_mode": scope["mode"], "audience_keys": scope["audience_keys"], + # Organisation audience is not assertion authorship. Keeping the + # trusted author in the identity prevents one member from replacing + # another member's provenance for the same organisation-visible claim. + "asserted_by_user_concept_id": scope["user_concept_id"], "subject_concept_id": subject_id, "predicate": storage_predicate, "object_kind": object_kind, @@ -193,55 +325,204 @@ def upsert_scoped_assertion( now = datetime.now(timezone.utc) provenance = { "asserted_by_user_concept_id": scope["user_concept_id"], - "organisation_concept_id": scope["organisation_concept_id"], - "namespace": scope["namespace"], + "organisation_concept_id": actor_context["organisation_concept_id"], + "namespace": actor_context["namespace"], "turn_id": str(turn_id or "").strip() or None, "capability_name": "upsert_scoped_assertion", "evidence": _normalise_evidence(evidence), } - update_fields: dict[str, Any] = { + insert_fields: dict[str, Any] = { + # Mongo's built-in _id uniqueness is the final concurrency boundary + # even if creation of the secondary assertion_id index was temporarily + # unavailable at process startup. + "_id": assertion_id, "schema_version": SCHEMA_VERSION, "assertion_id": assertion_id, "subject_concept_id": subject_id, "predicate": storage_predicate, "object_kind": object_kind, "scope": scope, - "provenance": provenance, "canonical_publication": False, "status": "asserted", + "created_at": now, "updated_at": now, + "provenance": provenance, } if object_kind == "text": - update_fields["object_text"] = { + insert_fields["object_text"] = { "text": object_text, "language": language_token, } - update_fields["object_concept_id"] = None + insert_fields["object_concept_id"] = None else: - update_fields["object_concept_id"] = object_concept_id - update_fields["object_text"] = None + insert_fields["object_concept_id"] = object_concept_id + insert_fields["object_text"] = None collection = get_scoped_knowledge_assertions_collection() if collection is None: raise RuntimeError("scoped assertion storage is unavailable") - existing = collection.find_one({"assertion_id": assertion_id}, {"_id": 1}) + try: + previous = collection.find_one_and_update( + {"assertion_id": assertion_id}, + {"$setOnInsert": insert_fields}, + upsert=True, + # BEFORE is None only for the operation that atomically created the + # assertion. A separate pre-read cannot provide this + # concurrency-safe changed result. + return_document=ReturnDocument.BEFORE, + ) + except DuplicateKeyError: + # A concurrent first writer may win on the deterministic built-in _id + # before this unindexed assertion_id query observes it. That is an + # idempotent replay, not a failed or indeterminate effect. + previous = collection.find_one({"assertion_id": assertion_id}) + if previous is None: + raise + created = previous is None + reactivated = None + if isinstance(previous, Mapping) and previous.get("status") == "retracted": + retraction_history_item = { + "retracted_at": previous.get("retracted_at"), + "retraction_provenance": previous.get("retraction_provenance"), + } + reassertion_provenance = { + "reasserted_by_user_concept_id": actor_context["user_concept_id"], + "organisation_concept_id": actor_context[ + "organisation_concept_id" + ], + "namespace": actor_context["namespace"], + "turn_id": str(turn_id or "").strip() or None, + "capability_name": "upsert_scoped_assertion", + "evidence": _normalise_evidence(evidence), + } + # Compare-and-set makes exactly one concurrent replay reactivate the + # tombstone. Original assertion provenance and logical scope remain + # immutable; a bounded history preserves the retraction being undone. + reactivated = collection.find_one_and_update( + { + "assertion_id": assertion_id, + "status": "retracted", + }, + { + "$set": { + "status": "asserted", + "updated_at": now, + "reasserted_at": now, + "reassertion_provenance": reassertion_provenance, + }, + "$unset": { + "retracted_at": "", + "retraction_provenance": "", + }, + "$push": { + "retraction_history": { + "$each": [retraction_history_item], + "$slice": -MAX_RETRACTION_HISTORY, + } + }, + }, + return_document=ReturnDocument.AFTER, + ) + persisted = reactivated or collection.find_one( + {"assertion_id": assertion_id} + ) + read_back = _serialise(persisted) + if read_back is None: + raise RuntimeError("scoped assertion write did not produce read-back") + return { + "success": True, + "effect_status": "succeeded", + "changed": bool(created or reactivated is not None), + "assertion_id": assertion_id, + "assertion": read_back, + "canonical_read_back": read_back, + "canonical_publication": False, + "storage_surface": STORAGE_SURFACE, + } + + +def retract_scoped_assertion( + *, + assertion_id: Any, + acting_user_concept_id: Any, + organisation_concept_id: Any = None, + namespace: Any = None, +) -> dict[str, Any]: + """Idempotently retract an assertion under its original author's authority. + + This recovery operation deliberately does not require current visibility of + the subject, predicate, or object: losing visibility must not strand an + assertion that the original author remains authorised to withdraw. + """ + + assertion_token = str(assertion_id or "").strip() + if not assertion_token.startswith("ska_"): + raise ValueError("assertion_id must be an exact scoped assertion ID") + actor_context = _resolve_actor_context( + acting_user_concept_id=acting_user_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + ) + user_id = actor_context["user_concept_id"] + org_id = actor_context["organisation_concept_id"] + scope_clauses: list[dict[str, Any]] = [{"scope.mode": "user"}] + if org_id: + scope_clauses.append( + { + "scope.mode": "organisation", + "scope.organisation_concept_id": org_id, + } + ) + authority_filter: dict[str, Any] = { + "assertion_id": assertion_token, + "provenance.asserted_by_user_concept_id": user_id, + "$or": scope_clauses, + } + now = datetime.now(timezone.utc) + retraction_provenance = { + "retracted_by_user_concept_id": user_id, + "organisation_concept_id": org_id, + "namespace": actor_context["namespace"], + "capability_name": "retract_scoped_assertion", + } + + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + raise RuntimeError("scoped assertion storage is unavailable") + retract_filter = { + **authority_filter, + "status": "asserted", + } persisted = collection.find_one_and_update( - {"assertion_id": assertion_id}, + retract_filter, { - "$set": update_fields, - "$setOnInsert": {"created_at": now}, + "$set": { + "status": "retracted", + "retracted_at": now, + "updated_at": now, + "retraction_provenance": retraction_provenance, + } }, - upsert=True, return_document=ReturnDocument.AFTER, ) + changed = persisted is not None + if persisted is None: + # This authority-bearing filter is intentionally reused for idempotent + # read-back. A missing or differently owned assertion yields the same + # generic denial and does not disclose whether the identifier exists. + persisted = collection.find_one(authority_filter) + if persisted is None: + raise PermissionError( + "scoped assertion is unavailable to the current actor" + ) read_back = _serialise(persisted) if read_back is None: - raise RuntimeError("scoped assertion write did not produce read-back") + raise RuntimeError("scoped assertion retraction did not produce read-back") return { "success": True, "effect_status": "succeeded", - "changed": existing is None, - "assertion_id": assertion_id, + "changed": changed, + "assertion_id": assertion_token, "assertion": read_back, "canonical_read_back": read_back, "canonical_publication": False, @@ -249,96 +530,563 @@ def upsert_scoped_assertion( } -def _audience_keys_for_actor( +def _resolve_read_actor( *, user_concept_id: str | None = None, organisation_concept_id: str | None = None, -) -> list[str]: - user_id = _clean_optional_concept_id( - user_concept_id or get_effective_user_concept_id() +) -> tuple[str | None, str | None]: + """Resolve an explicit trusted actor without contradicting ambient scope.""" + + ambient_user_id = _clean_optional_concept_id(get_effective_user_concept_id()) + ambient_org_id = _clean_optional_concept_id( + get_effective_organisation_concept_id() ) - org_id = _clean_optional_concept_id( - organisation_concept_id or get_effective_organisation_concept_id() + has_ambient_actor = bool(ambient_user_id or ambient_org_id) + + explicit_user_id = ( + _clean_concept_id(user_concept_id, "user_concept_id") + if str(user_concept_id or "").strip() + else None ) + explicit_org_id = ( + _clean_concept_id( + organisation_concept_id, + "organisation_concept_id", + ) + if str(organisation_concept_id or "").strip() + else None + ) + if ( + explicit_user_id + and has_ambient_actor + and explicit_user_id != ambient_user_id + ): + raise PermissionError( + "explicit user audience does not match the current trusted actor" + ) + if explicit_org_id and has_ambient_actor and explicit_org_id != ambient_org_id: + raise PermissionError( + "explicit organisation audience does not match the current trusted actor" + ) + + return ( + explicit_user_id or ambient_user_id, + explicit_org_id or ambient_org_id, + ) + + +def _audience_keys_for_actor( + *, + user_concept_id: str | None, + organisation_concept_id: str | None, +) -> list[str]: keys: list[str] = [] - if user_id: - keys.append(f"user:{user_id}") - if org_id: - keys.append(f"org:{org_id}") + if user_concept_id: + keys.append(f"user:{user_concept_id}") + if organisation_concept_id: + keys.append(f"org:{organisation_concept_id}") return keys -def list_visible_scoped_assertions( +def _normalise_subject_ids( + subject_concept_ids: Sequence[str] | None, +) -> list[str]: + ordered_subjects: list[str] = [] + seen: set[str] = set() + for item in subject_concept_ids or (): + if not isinstance(item, str) or not item.strip(): + continue + subject_id = _clean_concept_id(item, "subject_concept_ids item") + if subject_id in seen: + continue + seen.add(subject_id) + ordered_subjects.append(subject_id) + if len(ordered_subjects) > MAX_PAGE_SUBJECTS: + raise ValueError( + f"subject_concept_ids is limited to {MAX_PAGE_SUBJECTS} concepts" + ) + return ordered_subjects + + +def _assertion_visibility_concept_ids( + assertion: Mapping[str, Any], +) -> tuple[str, ...] | None: + """Return every concept whose current visibility is required for one row.""" + + subject_id = _clean_optional_concept_id(assertion.get("subject_concept_id")) + predicate_id = predicate_concept_id_for_storage(assertion.get("predicate")) + if not subject_id or not predicate_id: + return None + if assertion.get("object_kind") != "concept": + return (subject_id, predicate_id) + object_concept_id = _clean_optional_concept_id( + assertion.get("object_concept_id") + ) + if not object_concept_id: + return None + return (subject_id, predicate_id, object_concept_id) + + +def _visible_assertion_documents( + documents: Sequence[Mapping[str, Any]], +) -> list[Mapping[str, Any]]: + """Batch current visibility once for a bounded candidate window.""" + + candidates: list[tuple[Mapping[str, Any], tuple[str, ...]]] = [] + concept_ids: set[str] = set() + for document in documents: + required_ids = _assertion_visibility_concept_ids(document) + if required_ids is None: + continue + candidates.append((document, required_ids)) + concept_ids.update(required_ids) + visible_ids = filter_accessible_concept_ids(concept_ids) + return [ + document + for document, required_ids in candidates + if all(concept_id in visible_ids for concept_id in required_ids) + ] + + +def _normalise_page_bound( + value: Any, + *, + name: str, + minimum: int, + maximum: int, +) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be an integer") from exc + if parsed < minimum: + raise ValueError(f"{name} must be at least {minimum}") + return min(parsed, maximum) + + +def _build_assertion_query( + *, + audience_keys: Sequence[str], + subject_concept_ids: Sequence[str], + argument_concept_id: str | None, + predicates: Sequence[str] | None, + object_kind: str | None, + languages: Sequence[str] | None, +) -> dict[str, Any]: + query: dict[str, Any] = { + "scope.audience_keys": {"$in": list(audience_keys)}, + "status": "asserted", + } + if subject_concept_ids: + query["subject_concept_id"] = {"$in": list(subject_concept_ids)} + if argument_concept_id: + query["$or"] = [ + {"subject_concept_id": argument_concept_id}, + {"object_concept_id": argument_concept_id}, + ] + if predicates: + predicate_values: set[str] = set() + for item in predicates: + if not isinstance(item, str) or not item.strip(): + continue + predicate_value = item.strip() + predicate_values.add(predicate_value) + predicate_concept_id = predicate_concept_id_for_storage( + predicate_value + ) + if predicate_concept_id: + predicate_values.add(predicate_concept_id) + if predicate_value.startswith("#V#"): + storage_candidate = predicate_value[3:] + if ( + predicate_concept_id_for_storage(storage_candidate) + == predicate_value + ): + predicate_values.add(storage_candidate) + if predicate_values: + query["predicate"] = {"$in": sorted(predicate_values)} + if object_kind is not None: + if object_kind not in OBJECT_KINDS: + raise ValueError("object_kind must be 'text' or 'concept'") + query["object_kind"] = object_kind + if languages: + language_values = sorted( + { + str(item).strip() + for item in languages + if isinstance(item, str) and item.strip() + } + ) + if language_values: + query["object_kind"] = "text" + query["object_text.language"] = {"$in": language_values} + return query + + +def _empty_page( + *, + offset: int, + limit: int, + limit_per_subject: int | None, + subject_concept_ids: Sequence[str], +) -> dict[str, Any]: + per_subject = { + subject_id: { + "items": [], + "returned": 0, + "offset": offset, + "limit": limit_per_subject, + "has_more": False, + "next_offset": None, + "visibility_filtered": False, + "counts_are_lower_bounds": False, + } + for subject_id in subject_concept_ids + } + return { + "items": [], + "returned": 0, + "offset": offset, + "limit": limit, + "limit_per_subject": limit_per_subject, + "has_more": False, + "next_offset": None, + "truncated": False, + "visibility_filtered": False, + "counts_are_lower_bounds": False, + "per_subject": per_subject, + } + + +def list_visible_scoped_assertions_page( *, subject_concept_ids: Sequence[str] | None = None, argument_concept_id: str | None = None, predicates: Sequence[str] | None = None, object_kind: str | None = None, + languages: Sequence[str] | None = None, limit: int = 200, + offset: int = 0, + limit_per_subject: int | None = None, user_concept_id: str | None = None, organisation_concept_id: str | None = None, -) -> list[dict[str, Any]]: - """Return scoped assertions visible to the effective or explicit actor.""" +) -> dict[str, Any]: + """Return a bounded page visible to one effective or explicit trusted actor. - audience_keys = _audience_keys_for_actor( + ``offset`` advances only through rows whose referenced concepts are + currently visible. Inaccessible and retracted storage candidates are + deliberately absent from pagination, counts, continuation, and diagnostics. + Candidate scanning is bounded; when that bound cannot establish a safe + visible page, the read fails closed rather than exposing a misleading or + visibility-sensitive continuation. With ``limit_per_subject``, one + aggregate command independently bounds each requested subject so an + assertion-rich subject cannot starve another. + """ + + page_limit = _normalise_page_bound( + limit, + name="limit", + minimum=1, + maximum=MAX_PAGE_LIMIT, + ) + page_offset = _normalise_page_bound( + offset, + name="offset", + minimum=0, + maximum=2_147_483_647, + ) + per_subject_limit = ( + _normalise_page_bound( + limit_per_subject, + name="limit_per_subject", + minimum=1, + maximum=MAX_PAGE_LIMIT, + ) + if limit_per_subject is not None + else None + ) + requested_subjects = _normalise_subject_ids(subject_concept_ids) + if per_subject_limit is not None and not requested_subjects: + raise ValueError( + "limit_per_subject requires explicit subject_concept_ids" + ) + if per_subject_limit is not None: + candidate_bound = len(requested_subjects) * ( + page_offset + per_subject_limit + 1 + ) + if candidate_bound > MAX_PAGE_CANDIDATES: + raise ValueError( + "per-subject visible page exceeds the bounded candidate limit " + f"of {MAX_PAGE_CANDIDATES}" + ) + + actor_user_id, actor_org_id = _resolve_read_actor( user_concept_id=user_concept_id, organisation_concept_id=organisation_concept_id, ) + audience_keys = _audience_keys_for_actor( + user_concept_id=actor_user_id, + organisation_concept_id=actor_org_id, + ) if not audience_keys: - return [] - query: dict[str, Any] = { - "scope.audience_keys": {"$in": audience_keys}, - "status": "asserted", - } - if subject_concept_ids: - visible_subjects = sorted( - { - str(item).strip() - for item in subject_concept_ids - if isinstance(item, str) - and item.strip() - and can_access_concept(item.strip()) - } + return _empty_page( + offset=page_offset, + limit=page_limit, + limit_per_subject=per_subject_limit, + subject_concept_ids=requested_subjects, ) - if not visible_subjects: - return [] - query["subject_concept_id"] = {"$in": visible_subjects} - if argument_concept_id: - concept_id = _clean_concept_id(argument_concept_id, "argument_concept_id") - if not can_access_concept(concept_id): - return [] - query["$or"] = [ - {"subject_concept_id": concept_id}, - {"object_concept_id": concept_id}, + + with override_current_actor(actor_user_id, actor_org_id): + resolved_argument_id = ( + _clean_concept_id( + argument_concept_id, + "argument_concept_id", + ) + if argument_concept_id + else None + ) + preflight_concept_ids = { + *requested_subjects, + *([resolved_argument_id] if resolved_argument_id else []), + } + visible_preflight_ids = filter_accessible_concept_ids( + preflight_concept_ids + ) + visible_subjects = [ + subject_id + for subject_id in requested_subjects + if subject_id in visible_preflight_ids ] - if predicates: - predicate_values = sorted( - { - str(item).strip() - for item in predicates - if isinstance(item, str) and item.strip() + if ( + resolved_argument_id + and resolved_argument_id not in visible_preflight_ids + ): + return _empty_page( + offset=page_offset, + limit=page_limit, + limit_per_subject=per_subject_limit, + subject_concept_ids=requested_subjects, + ) + if requested_subjects and not visible_subjects: + return _empty_page( + offset=page_offset, + limit=page_limit, + limit_per_subject=per_subject_limit, + subject_concept_ids=requested_subjects, + ) + + query = _build_assertion_query( + audience_keys=audience_keys, + subject_concept_ids=visible_subjects, + argument_concept_id=resolved_argument_id, + predicates=predicates, + object_kind=object_kind, + languages=languages, + ) + collection = get_scoped_knowledge_assertions_collection() + if collection is None: + return _empty_page( + offset=page_offset, + limit=page_limit, + limit_per_subject=per_subject_limit, + subject_concept_ids=requested_subjects, + ) + + if per_subject_limit is None: + raw_candidates = list( + collection.find(query) + .sort([("updated_at", -1), ("assertion_id", 1)]) + .limit(MAX_PAGE_CANDIDATES + 1) + ) + raw_window_exhausted = len(raw_candidates) > MAX_PAGE_CANDIDATES + visible_candidates = _visible_assertion_documents( + raw_candidates[:MAX_PAGE_CANDIDATES] + ) + visible_target = page_offset + page_limit + 1 + if ( + raw_window_exhausted + and len(visible_candidates) < visible_target + ): + raise RuntimeError( + "scoped assertion page could not be evaluated within the " + "bounded visibility window" + ) + has_more = len(visible_candidates) > page_offset + page_limit + page_candidates = visible_candidates[ + page_offset : page_offset + page_limit + ] + visible_items = [ + serialised + for document in page_candidates + if (serialised := _serialise(document)) is not None + ] + return { + "items": visible_items, + "returned": len(visible_items), + "offset": page_offset, + "limit": page_limit, + "limit_per_subject": None, + "has_more": has_more, + "next_offset": ( + page_offset + page_limit if has_more else None + ), + "truncated": has_more, + "visibility_filtered": False, + "counts_are_lower_bounds": has_more, + "per_subject": {}, } + + raw_by_subject: dict[str, list[Mapping[str, Any]]] = { + subject_id: [] for subject_id in requested_subjects + } + raw_window_exhausted_by_subject: dict[str, bool] = { + subject_id: False for subject_id in requested_subjects + } + if visible_subjects: + candidate_limit_per_subject = ( + MAX_PAGE_CANDIDATES // len(visible_subjects) + ) + branch_limit = candidate_limit_per_subject + 1 + + def _subject_pipeline(subject_id: str) -> list[dict[str, Any]]: + subject_query = dict(query) + subject_query["subject_concept_id"] = subject_id + return [ + {"$match": subject_query}, + { + "$sort": { + "updated_at": -1, + "assertion_id": 1, + } + }, + {"$limit": branch_limit}, + ] + + pipeline = _subject_pipeline(visible_subjects[0]) + collection_name = str( + getattr(collection, "name", STORAGE_SURFACE) + or STORAGE_SURFACE + ) + for subject_id in visible_subjects[1:]: + pipeline.append( + { + "$unionWith": { + "coll": collection_name, + "pipeline": _subject_pipeline(subject_id), + } + } + ) + for document in collection.aggregate(pipeline): + if not isinstance(document, Mapping): + continue + document_subject = str( + document.get("subject_concept_id") or "" + ).strip() + bucket = raw_by_subject.get(document_subject) + if bucket is not None: + bucket.append(document) + + visibility_candidates: list[Mapping[str, Any]] = [] + for subject_id in visible_subjects: + raw_subject_candidates = raw_by_subject[subject_id] + raw_window_exhausted_by_subject[subject_id] = ( + len(raw_subject_candidates) > candidate_limit_per_subject + ) + visibility_candidates.extend( + raw_subject_candidates[:candidate_limit_per_subject] + ) + visible_documents = _visible_assertion_documents( + visibility_candidates ) - if predicate_values: - query["predicate"] = {"$in": predicate_values} - if object_kind is not None: - if object_kind not in OBJECT_KINDS: - raise ValueError("object_kind must be 'text' or 'concept'") - query["object_kind"] = object_kind + visible_by_subject: dict[str, list[Mapping[str, Any]]] = { + subject_id: [] for subject_id in requested_subjects + } + for document in visible_documents: + document_subject = str( + document.get("subject_concept_id") or "" + ).strip() + bucket = visible_by_subject.get(document_subject) + if bucket is not None: + bucket.append(document) - collection = get_scoped_knowledge_assertions_collection() - if collection is None: - return [] - cursor = ( - collection.find(query) - .sort([("updated_at", -1), ("assertion_id", 1)]) - .limit(max(1, min(int(limit), 1000))) - ) - return [ - serialised - for document in cursor - if (serialised := _serialise(document)) is not None - ] + flattened_items: list[dict[str, Any]] = [] + per_subject: dict[str, dict[str, Any]] = {} + any_more = False + for subject_id in requested_subjects: + visible_subject_candidates = visible_by_subject[subject_id] + visible_target = page_offset + per_subject_limit + 1 + if ( + raw_window_exhausted_by_subject[subject_id] + and len(visible_subject_candidates) < visible_target + ): + raise RuntimeError( + "scoped assertion page could not be evaluated within the " + "bounded visibility window" + ) + has_more = ( + len(visible_subject_candidates) + > page_offset + per_subject_limit + ) + page_candidates = visible_subject_candidates[ + page_offset : page_offset + per_subject_limit + ] + visible_items = [ + serialised + for document in page_candidates + if (serialised := _serialise(document)) is not None + ] + any_more = any_more or has_more + flattened_items.extend(visible_items) + per_subject[subject_id] = { + "items": visible_items, + "returned": len(visible_items), + "offset": page_offset, + "limit": per_subject_limit, + "has_more": has_more, + "next_offset": ( + page_offset + per_subject_limit if has_more else None + ), + "visibility_filtered": False, + "counts_are_lower_bounds": has_more, + } + return { + "items": flattened_items, + "returned": len(flattened_items), + "offset": page_offset, + "limit": page_limit, + "limit_per_subject": per_subject_limit, + "has_more": any_more, + "next_offset": None, + "truncated": any_more, + "visibility_filtered": False, + "counts_are_lower_bounds": any_more, + "per_subject": per_subject, + } + + +def list_visible_scoped_assertions( + *, + subject_concept_ids: Sequence[str] | None = None, + argument_concept_id: str | None = None, + predicates: Sequence[str] | None = None, + object_kind: str | None = None, + languages: Sequence[str] | None = None, + limit: int = 200, + user_concept_id: str | None = None, + organisation_concept_id: str | None = None, +) -> list[dict[str, Any]]: + """Compatibility wrapper returning only the bounded page items.""" + + return list_visible_scoped_assertions_page( + subject_concept_ids=subject_concept_ids, + argument_concept_id=argument_concept_id, + predicates=predicates, + object_kind=object_kind, + languages=languages, + limit=limit, + user_concept_id=user_concept_id, + organisation_concept_id=organisation_concept_id, + )["items"] def scoped_text_assertion_to_relation_row( @@ -356,8 +1104,9 @@ def scoped_text_assertion_to_relation_row( "lang": object_text.get("language"), "text_value_id": None, "predicate": assertion.get("predicate"), - "relation_id": assertion_id or None, + "relation_id": None, "assertion_id": assertion_id or None, + "row_kind": "scoped_assertion", "context": { "assertion_scope": assertion.get("scope"), "canonical_publication": False, @@ -376,6 +1125,8 @@ def scoped_text_assertion_to_relation_row( __all__ = [ "list_visible_scoped_assertions", + "list_visible_scoped_assertions_page", + "retract_scoped_assertion", "scoped_text_assertion_to_relation_row", "upsert_scoped_assertion", ] diff --git a/src/backend/services/scoped_rag_authority_service.py b/src/backend/services/scoped_rag_authority_service.py new file mode 100644 index 000000000..0ff9987f4 --- /dev/null +++ b/src/backend/services/scoped_rag_authority_service.py @@ -0,0 +1,242 @@ +"""Live authority checks for scoped assertions returned from the RAG index.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from bson import ObjectId +from bson.errors import InvalidId + +from ..db.mongo_client import get_scoped_knowledge_assertions_collection +from ..db.repositories.text_value_repository import TextRelationsRepository +from ..security.access_control import ( + filter_accessible_concept_ids, + override_current_actor, +) +from .text_relation_predicate_validation_service import ( + predicate_concept_id_for_storage, +) + + +def _clean_concept_id(value: Any) -> str | None: + token = str(value or "").strip() + return token if token.startswith("#V#") else None + + +def _clean_assertion_id(value: Any) -> str | None: + token = str(value or "").strip() + return token if token.startswith("ska_") else None + + +def _clean_relation_id(value: Any) -> str | None: + token = str(value or "").strip() + return token or None + + +def _candidate_kind(metadata: Mapping[str, Any]) -> str | None: + kind = str(metadata.get("type") or "").strip() + source = str(metadata.get("source") or "").strip() + if kind == "scoped_knowledge_assertion" or source == ( + "scoped_knowledge_assertion" + ): + return "scoped_knowledge_assertion" + if kind == "text_relation" or source == "vontology_text_relation": + return "text_relation" + return None + + +def _relation_lookup_values(relation_ids: Sequence[str]) -> list[Any]: + values: list[Any] = [] + seen: set[tuple[type[Any], str]] = set() + for relation_id in relation_ids: + candidates: list[Any] = [relation_id] + try: + candidates.insert(0, ObjectId(relation_id)) + except (InvalidId, TypeError): + pass + for candidate in candidates: + key = (type(candidate), str(candidate)) + if key in seen: + continue + seen.add(key) + values.append(candidate) + return values + + +def current_authorised_rag_candidate_keys( + metadata_rows: Sequence[Mapping[str, Any]], + *, + user_concept_id: str | None, + organisation_concept_id: str | None, +) -> set[tuple[str, str]]: + """Return live-authorised canonical keys for knowledge RAG candidates.""" + + scoped_metadata: dict[str, Mapping[str, Any]] = {} + relation_metadata: dict[str, Mapping[str, Any]] = {} + for metadata in metadata_rows: + kind = _candidate_kind(metadata) + if kind == "scoped_knowledge_assertion": + assertion_id = _clean_assertion_id(metadata.get("assertion_id")) + if assertion_id: + scoped_metadata[assertion_id] = metadata + elif kind == "text_relation": + relation_id = _clean_relation_id(metadata.get("relation_id")) + if relation_id: + relation_metadata[relation_id] = metadata + if not scoped_metadata and not relation_metadata: + return set() + + actor_user_id = _clean_concept_id(user_concept_id) + actor_org_id = _clean_concept_id(organisation_concept_id) + audience_keys = [] + if actor_user_id: + audience_keys.append(f"user:{actor_user_id}") + if actor_org_id: + audience_keys.append(f"org:{actor_org_id}") + if not audience_keys: + return set() + + current_rows: dict[ + tuple[str, str], + tuple[str, str, str], + ] = {} + concept_ids: set[str] = set() + + if relation_metadata: + relations = TextRelationsRepository.find( + { + "_id": { + "$in": _relation_lookup_values( + list(relation_metadata) + ) + } + }, + { + "_id": 1, + "subject_concept_id": 1, + "predicate": 1, + }, + ) + for relation in relations: + if not isinstance(relation, Mapping): + continue + relation_id = _clean_relation_id(relation.get("_id")) + subject_id = _clean_concept_id( + relation.get("subject_concept_id") + ) + predicate = str(relation.get("predicate") or "").strip() + predicate_id = predicate_concept_id_for_storage(predicate) + if ( + not relation_id + or relation_id not in relation_metadata + or not subject_id + or not predicate + or not predicate_id + ): + continue + current_rows[("text_relation", relation_id)] = ( + subject_id, + predicate, + predicate_id, + ) + concept_ids.update((subject_id, predicate_id)) + + collection = ( + get_scoped_knowledge_assertions_collection() + if scoped_metadata + else None + ) + if collection is not None: + assertions = collection.find( + { + "assertion_id": {"$in": sorted(scoped_metadata)}, + "status": "asserted", + "object_kind": "text", + "scope.audience_keys": {"$in": audience_keys}, + }, + { + "_id": 0, + "assertion_id": 1, + "subject_concept_id": 1, + "predicate": 1, + }, + ) + for assertion in assertions: + if not isinstance(assertion, Mapping): + continue + assertion_id = _clean_assertion_id( + assertion.get("assertion_id") + ) + subject_id = _clean_concept_id( + assertion.get("subject_concept_id") + ) + predicate = str(assertion.get("predicate") or "").strip() + predicate_id = predicate_concept_id_for_storage(predicate) + if ( + not assertion_id + or not subject_id + or not predicate + or not predicate_id + ): + continue + current_rows[("scoped_knowledge_assertion", assertion_id)] = ( + subject_id, + predicate, + predicate_id, + ) + concept_ids.update((subject_id, predicate_id)) + + with override_current_actor(actor_user_id, actor_org_id): + visible_concept_ids = filter_accessible_concept_ids(concept_ids) + + allowed: set[tuple[str, str]] = set() + for key, (subject_id, predicate, predicate_id) in current_rows.items(): + kind, candidate_id = key + metadata = ( + relation_metadata.get(candidate_id) + if kind == "text_relation" + else scoped_metadata.get(candidate_id) + ) + if metadata is None: + continue + if str(metadata.get("subject_concept_id") or "").strip() != subject_id: + continue + if str(metadata.get("predicate") or "").strip() != predicate: + continue + if subject_id not in visible_concept_ids: + continue + if predicate_id not in visible_concept_ids: + continue + allowed.add(key) + return allowed + + +def current_authorised_scoped_assertion_ids( + metadata_rows: Sequence[Mapping[str, Any]], + *, + user_concept_id: str | None, + organisation_concept_id: str | None, +) -> set[str]: + """Return candidate IDs that remain asserted, visible, and in audience. + + RAG is a derived index, so its metadata is only a candidate locator. This + function re-reads the scoped assertion store and batches current Vontology + visibility checks for both the assertion subject and represented predicate. + """ + + return { + candidate_id + for kind, candidate_id in current_authorised_rag_candidate_keys( + metadata_rows, + user_concept_id=user_concept_id, + organisation_concept_id=organisation_concept_id, + ) + if kind == "scoped_knowledge_assertion" + } + + +__all__ = [ + "current_authorised_rag_candidate_keys", + "current_authorised_scoped_assertion_ids", +] diff --git a/src/backend/services/testing_theory_service.py b/src/backend/services/testing_theory_service.py index 07c344257..8728d52a0 100644 --- a/src/backend/services/testing_theory_service.py +++ b/src/backend/services/testing_theory_service.py @@ -470,7 +470,12 @@ def _canonical_text_exists( predicate: str, target_text: str, ) -> bool: - rows = get_texts_for_concept(source_id, predicate=predicate, limit=100) + rows = get_texts_for_concept( + source_id, + predicate=predicate, + limit=100, + context_view="base_publication", + ) expected = target_text.strip() return any(_safe_str(row.get("text")) == expected for row in rows if isinstance(row, Mapping)) diff --git a/src/backend/services/text_relation_predicate_validation_service.py b/src/backend/services/text_relation_predicate_validation_service.py index 92fdbb130..299bdc50c 100644 --- a/src/backend/services/text_relation_predicate_validation_service.py +++ b/src/backend/services/text_relation_predicate_validation_service.py @@ -49,6 +49,19 @@ def __init__( self.suggestions = list(suggestions or []) +def predicate_concept_id_for_storage(predicate: Any) -> str | None: + """Return the represented predicate concept for one stored text predicate.""" + + predicate_text = str(predicate or "").strip() + if predicate_text in _CORE_TEXT_PREDICATE_CONCEPT_BY_STORAGE: + return _CORE_TEXT_PREDICATE_CONCEPT_BY_STORAGE[predicate_text] + if predicate_text in _CORE_TEXT_PREDICATE_STORAGE_BY_CONCEPT_ID: + return predicate_text + if predicate_text.startswith("#V#"): + return predicate_text + return None + + def _relationship_values(doc: Mapping[str, Any], key: str) -> set[str]: relationships = doc.get("relationships") if not isinstance(relationships, Mapping): @@ -177,6 +190,7 @@ def resolve_text_relation_predicate_for_write( __all__ = [ + "predicate_concept_id_for_storage", "TextRelationPredicateResolution", "TextRelationPredicateResolutionError", "resolve_text_relation_predicate_for_write", diff --git a/src/backend/services/text_value_service.py b/src/backend/services/text_value_service.py index 6e237591c..8f20d2dd8 100644 --- a/src/backend/services/text_value_service.py +++ b/src/backend/services/text_value_service.py @@ -7,7 +7,17 @@ from __future__ import annotations import re -from typing import Dict, Any, List, Optional, Literal, Sequence, cast, Tuple +from typing import ( + Any, + Dict, + List, + Literal, + Mapping, + Optional, + Sequence, + Tuple, + cast, +) from datetime import datetime, timezone from bson import ObjectId from bson.errors import InvalidId @@ -35,19 +45,46 @@ _EVENT_TYPE_TEXT_RELATION_UPDATED = "text_relation.updated" _EVENT_TYPE_TEXT_RELATION_DELETED = "text_relation.deleted" +TextContextView = Literal["base_publication", "actor_effective"] +_TEXT_CONTEXT_VIEWS = frozenset({"base_publication", "actor_effective"}) + def _now() -> datetime: return datetime.now(timezone.utc) def _iso_utc(value: Any) -> str | None: - if isinstance(value, datetime): - return value.isoformat() - if isinstance(value, str) and value.strip(): - return value.strip() + resolved = _as_utc_datetime(value) + if resolved is not None: + return resolved.isoformat() return None +def _as_utc_datetime(value: Any) -> datetime | None: + resolved: datetime + if isinstance(value, datetime): + resolved = value + elif isinstance(value, str) and value.strip(): + try: + resolved = datetime.fromisoformat( + value.strip().replace("Z", "+00:00") + ) + except ValueError: + return None + else: + return None + if resolved.tzinfo is None: + resolved = resolved.replace(tzinfo=timezone.utc) + return resolved.astimezone(timezone.utc) + + +def _row_timestamp(row: Dict[str, Any]) -> datetime: + resolved = _as_utc_datetime( + row.get("relation_updated_at") or row.get("relation_created_at") + ) + return resolved or datetime.min.replace(tzinfo=timezone.utc) + + def _invalidate_stats_for_predicate_change(predicate: str | None) -> None: """Best-effort stats invalidation for ontology predicate extent changes.""" @@ -399,10 +436,14 @@ def get_texts_for_concept( limit: int = 50, recent_first: bool = False, max_time_ms: Optional[int] = None, + *, + context_view: TextContextView = "base_publication", ) -> List[Dict[str, Any]]: - """Fetch linked TextValues for a concept, optionally filtered by predicate and lang. + """Fetch text assertions in an explicit semantic view. - Returns a list of { text, lang, text_value_id, predicate, relation_id, context }. + ``base_publication`` preserves the repository's historical relation-store + read. ``actor_effective`` overlays scoped assertions visible to the current + trusted actor. Storage location alone must not choose the view. """ rows_by_concept = get_texts_for_concepts( [subject_concept_id], @@ -411,6 +452,7 @@ def get_texts_for_concept( limit_per_concept=limit, recent_first=recent_first, max_time_ms=max_time_ms, + context_view=context_view, ) return rows_by_concept.get(subject_concept_id, []) @@ -425,13 +467,19 @@ def get_texts_for_concepts( recent_first: bool = False, max_time_ms: Optional[int] = None, query_metadata: Optional[Dict[str, Any]] = None, + context_view: TextContextView = "base_publication", ) -> Dict[str, List[Dict[str, Any]]]: - """Fetch linked TextValues for many concepts with one relation/text join. + """Fetch text assertions for many concepts in an explicit semantic view. Returns rows grouped by concept id. This is deliberately generic support for Vontology-heavy paths such as workflow publication/read-back, where many step concepts need their policy text relations at once. """ + if context_view not in _TEXT_CONTEXT_VIEWS: + raise ValueError( + "context_view must be 'base_publication' or 'actor_effective'" + ) + candidate_subject_ids: List[str] = [] seen_subject_ids: set[str] = set() for raw_id in subject_concept_ids: @@ -457,6 +505,10 @@ def get_texts_for_concepts( "raw_relation_count": 0, "relation_query_limit": 0, "relation_query_truncated": False, + "context_view": context_view, + "scoped_assertion_count": 0, + "scoped_query_limit": 0, + "scoped_query_truncated": False, } ) return {} @@ -503,7 +555,7 @@ def get_texts_for_concepts( rows_by_concept: Dict[str, List[Dict[str, Any]]] = { subject_id: [] for subject_id in ordered_subject_ids } - rows = ( + base_rows = ( _resolve_text_rows_from_relations( relations, lang=lang, @@ -513,14 +565,34 @@ def get_texts_for_concepts( if relations else [] ) - from .scoped_assertion_service import ( - list_visible_scoped_assertions, - scoped_text_assertion_to_relation_row, - ) + scoped_rows: List[Dict[str, Any]] = [] + scoped_query_limit = 0 + scoped_query_truncated = False + scoped_visibility_filtered = False + scoped_counts_are_lower_bounds = False + scoped_per_concept: Dict[str, Dict[str, Any]] = {} + if context_view == "actor_effective": + from .scoped_assertion_service import ( + MAX_PAGE_CANDIDATES, + MAX_PAGE_LIMIT, + MAX_PAGE_SUBJECTS, + list_visible_scoped_assertions_page, + scoped_text_assertion_to_relation_row, + ) - scoped_assertions = list_visible_scoped_assertions( - subject_concept_ids=ordered_subject_ids, - predicates=( + requested_per_concept = max(limit_per_concept, 1) + scoped_per_concept_limit = min( + requested_per_concept, + MAX_PAGE_LIMIT, + ) + max_subjects_per_query = min( + MAX_PAGE_SUBJECTS, + max( + 1, + MAX_PAGE_CANDIDATES // (scoped_per_concept_limit + 1), + ), + ) + scoped_predicates = ( [predicate] if predicate else [ @@ -529,24 +601,92 @@ def get_texts_for_concepts( if isinstance(item, str) and str(item).strip() ] or None - ), - object_kind="text", - limit=max(len(ordered_subject_ids) * max(limit_per_concept, 1), 1), - ) - for assertion in scoped_assertions: - scoped_row = scoped_text_assertion_to_relation_row(assertion) - if scoped_row is None: - continue - if lang and scoped_row.get("lang") != lang: - continue - rows.append(scoped_row) + ) + if requested_per_concept > scoped_per_concept_limit: + scoped_query_truncated = True + scoped_counts_are_lower_bounds = True + for batch_start in range( + 0, + len(ordered_subject_ids), + max_subjects_per_query, + ): + subject_batch = ordered_subject_ids[ + batch_start : batch_start + max_subjects_per_query + ] + scoped_page = list_visible_scoped_assertions_page( + subject_concept_ids=subject_batch, + predicates=scoped_predicates, + object_kind="text", + languages=[lang] if lang else None, + limit=min( + MAX_PAGE_LIMIT, + max(len(subject_batch) * scoped_per_concept_limit, 1), + ), + limit_per_subject=scoped_per_concept_limit, + ) + scoped_query_limit += ( + len(subject_batch) * scoped_per_concept_limit + ) + scoped_query_truncated = bool( + scoped_query_truncated or scoped_page.get("truncated") + ) + scoped_visibility_filtered = bool( + scoped_visibility_filtered + or scoped_page.get("visibility_filtered") + ) + scoped_counts_are_lower_bounds = bool( + scoped_counts_are_lower_bounds + or scoped_page.get("counts_are_lower_bounds") + ) + page_by_subject = scoped_page.get("per_subject") + if isinstance(page_by_subject, Mapping): + for subject_id, subject_page in page_by_subject.items(): + if not isinstance(subject_page, Mapping): + continue + scoped_per_concept[str(subject_id)] = { + "returned": int(subject_page.get("returned") or 0), + "limit": subject_page.get("limit"), + "has_more": bool(subject_page.get("has_more")), + "visibility_filtered": bool( + subject_page.get("visibility_filtered") + ), + "counts_are_lower_bounds": bool( + subject_page.get("counts_are_lower_bounds") + ), + } + for assertion in scoped_page.get("items") or (): + scoped_row = scoped_text_assertion_to_relation_row(assertion) + if scoped_row is None: + continue + scoped_rows.append(scoped_row) + + if query_metadata is not None: + query_metadata.update( + { + "context_view": context_view, + "scoped_assertion_count": len(scoped_rows), + "scoped_query_limit": scoped_query_limit, + "scoped_query_truncated": scoped_query_truncated, + "scoped_visibility_filtered": scoped_visibility_filtered, + "scoped_counts_are_lower_bounds": ( + scoped_counts_are_lower_bounds + ), + "scoped_per_concept": scoped_per_concept, + } + ) if recent_first: - rows.sort( - key=lambda row: str( - row.get("relation_updated_at") or row.get("relation_created_at") or "" - ), - reverse=True, + rows = [*base_rows, *scoped_rows] + rows.sort(key=_row_timestamp, reverse=True) + else: + # Direct actor-scoped deltas are surfaced before the base rows in a + # bounded effective view so a small limit cannot make the actor's own + # assertion disappear. This is retrieval priority, not logical + # override: both row kinds remain explicitly labelled. + rows = ( + [*scoped_rows, *base_rows] + if context_view == "actor_effective" + else base_rows ) for row in rows: subject_concept_id = row.get("subject_concept_id") @@ -620,6 +760,7 @@ def _resolve_text_rows_from_relations( "text_value_id": str(tv.get("_id")), "predicate": r.get("predicate"), "relation_id": str(r.get("_id")) if r.get("_id") else None, + "row_kind": "base_text_relation", "context": r.get("context", {}), "provenance": tv.get("provenance", {}), "text_value_created_at": _iso_utc(tv.get("created_at")), @@ -1066,16 +1207,23 @@ def get_text_relations_summary( predicates: Optional[List[str]] = None, languages: Optional[List[str]] = None, max_relation_ids_per_group: int = 25, + context_view: TextContextView = "base_publication", ) -> Dict[str, Any]: - """Return counts + relation IDs grouped by predicate/language. + """Return bounded counts and typed row IDs by predicate/language. This is intentionally "lightweight": it does not return full text bodies. Notes: - Text relation documents do not store language directly; language is derived from the referenced text_values documents. - - `languages` is therefore a post-filter applied after resolving text value lang. + - Base `languages` filtering is applied after resolving text value language. + - Actor-effective scoped counts are lower bounds when + ``scoped_query_truncated`` is true. """ + if context_view not in _TEXT_CONTEXT_VIEWS: + raise ValueError( + "context_view must be 'base_publication' or 'actor_effective'" + ) if not can_access_concept(subject_concept_id): raise PermissionError("User cannot view text for an inaccessible concept") @@ -1149,7 +1297,10 @@ def get_text_relations_summary( "language": lang, "count": 0, "relation_ids": [], + "assertion_ids": [], "latest_relation_id": None, + "latest_assertion_id": None, + "latest_row_kind": None, "latest_updated_at": None, } grouped[key] = bucket @@ -1160,21 +1311,39 @@ def get_text_relations_summary( if len(bucket["relation_ids"]) < max_relation_ids_per_group: bucket["relation_ids"].append(rel_id) - updated_at = rel.get("updated_at") or rel.get("created_at") - if updated_at is not None: + updated_at = _iso_utc(rel.get("updated_at") or rel.get("created_at")) + if updated_at: current_latest = bucket.get("latest_updated_at") if current_latest is None or updated_at > current_latest: bucket["latest_updated_at"] = updated_at bucket["latest_relation_id"] = rel_id + bucket["latest_assertion_id"] = None + bucket["latest_row_kind"] = "base_text_relation" + + scoped_assertions: List[Dict[str, Any]] = [] + scoped_query_truncated = False + scoped_visibility_filtered = False + scoped_counts_are_lower_bounds = False + if context_view == "actor_effective": + from .scoped_assertion_service import ( + list_visible_scoped_assertions_page, + ) - from .scoped_assertion_service import list_visible_scoped_assertions - - scoped_assertions = list_visible_scoped_assertions( - subject_concept_ids=[subject_concept_id], - predicates=predicates, - object_kind="text", - limit=1000, - ) + scoped_page = list_visible_scoped_assertions_page( + subject_concept_ids=[subject_concept_id], + predicates=predicates, + object_kind="text", + languages=languages, + limit=1000, + ) + scoped_assertions = list(scoped_page.get("items") or ()) + scoped_query_truncated = bool(scoped_page.get("truncated")) + scoped_visibility_filtered = bool( + scoped_page.get("visibility_filtered") + ) + scoped_counts_are_lower_bounds = bool( + scoped_page.get("counts_are_lower_bounds") + ) for assertion in scoped_assertions: object_text = assertion.get("object_text") if not isinstance(object_text, dict): @@ -1191,40 +1360,43 @@ def get_text_relations_summary( "language": lang, "count": 0, "relation_ids": [], + "assertion_ids": [], "latest_relation_id": None, + "latest_assertion_id": None, + "latest_row_kind": None, "latest_updated_at": None, } grouped[key] = bucket bucket["count"] += 1 bucket["scoped_count"] = int(bucket.get("scoped_count") or 0) + 1 assertion_id = str(assertion.get("assertion_id") or "") - if assertion_id and len(bucket["relation_ids"]) < max_relation_ids_per_group: - bucket["relation_ids"].append(assertion_id) - updated_at = assertion.get("updated_at") or assertion.get("created_at") + if assertion_id and len(bucket["assertion_ids"]) < max_relation_ids_per_group: + bucket["assertion_ids"].append(assertion_id) + updated_at = _iso_utc( + assertion.get("updated_at") or assertion.get("created_at") + ) current_latest = bucket.get("latest_updated_at") - if updated_at is not None and ( - current_latest is None or str(updated_at) > str(current_latest) - ): + if updated_at and (current_latest is None or updated_at > current_latest): bucket["latest_updated_at"] = updated_at - bucket["latest_relation_id"] = assertion_id + bucket["latest_relation_id"] = None + bucket["latest_assertion_id"] = assertion_id + bucket["latest_row_kind"] = "scoped_assertion" summary_items = list(grouped.values()) summary_items.sort( key=lambda x: (x.get("predicate") or "", x.get("language") or "") ) - for item in summary_items: - # Make datetime JSON-friendly - dt = item.get("latest_updated_at") - if isinstance(dt, datetime): - item["latest_updated_at"] = dt.isoformat() - return { "success": True, "concept_id": subject_concept_id, + "context_view": context_view, "groups": summary_items, "groups_found": len(summary_items), "total_relations_scanned": len(rels) + len(scoped_assertions), "scoped_relations_scanned": len(scoped_assertions), + "scoped_query_truncated": scoped_query_truncated, + "scoped_visibility_filtered": scoped_visibility_filtered, + "counts_are_lower_bounds": scoped_counts_are_lower_bounds, "max_relation_ids_per_group": max_relation_ids_per_group, } diff --git a/src/utilities/reindex_text_relations.py b/src/utilities/reindex_text_relations.py index f6eb6b3c9..2ecce3e28 100644 --- a/src/utilities/reindex_text_relations.py +++ b/src/utilities/reindex_text_relations.py @@ -4,13 +4,13 @@ import logging from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Iterable, List, Optional, Sequence +from typing import Any, List, Optional, Sequence from src.backend.db.repositories.text_value_repository import TextRelationsRepository from src.backend.services.rag_service import RAGBackendUnavailable, get_rag_service from src.backend.services.rag_text_relation_sync_service import ( TextRelationRagDoc, - collect_text_relation_docs_for_namespace, + iter_text_relation_docs_for_namespace, ) logger = logging.getLogger(__name__) @@ -140,7 +140,10 @@ def parse_args(argv: Optional[Sequence[str]] = None) -> ReindexArgs: "--page-size", type=int, default=5000, - help="How many raw relations to scan per DB page (default: 5000).", + help=( + "Maximum source rows retained per one-pass scan batch " + "(default: 5000)." + ), ) parser.add_argument( "--batch-size", @@ -152,7 +155,7 @@ def parse_args(argv: Optional[Sequence[str]] = None) -> ReindexArgs: "--scan-limit", type=int, default=0, - help="Maximum raw relations to scan (0 = no limit).", + help="Maximum combined base/scoped documents to scan (0 = no limit).", ) args = parser.parse_args(list(argv) if argv is not None else None) @@ -186,14 +189,6 @@ def parse_args(argv: Optional[Sequence[str]] = None) -> ReindexArgs: ) -def _chunks( - items: List[TextRelationRagDoc], n: int -) -> Iterable[List[TextRelationRagDoc]]: - step = max(int(n), 1) - for i in range(0, len(items), step): - yield items[i : i + step] - - def run_reindex(args: ReindexArgs) -> int: rel_filter: dict[str, Any] = {} if args.predicates: @@ -204,15 +199,12 @@ def run_reindex(args: ReindexArgs) -> int: rel_filter["subject_concept_id"] = {"$in": list(args.concept_ids)} total_raw = TextRelationsRepository.count_documents(rel_filter) - raw_to_scan = total_raw - if args.scan_limit: - raw_to_scan = min(raw_to_scan, args.scan_limit) - logger.info( - "Starting reindex: namespace=%s raw_total=%s raw_scan=%s dry_run=%s", + "Starting reindex: namespace=%s base_raw_total=%s combined_scan_limit=%s " + "dry_run=%s", args.namespace, total_raw, - raw_to_scan, + args.scan_limit or "unbounded", args.dry_run, ) @@ -227,53 +219,77 @@ def run_reindex(args: ReindexArgs) -> int: total_candidates = 0 added = 0 failed = 0 + deleted = 0 + + def _delete_stale_batch(stale_ids: Sequence[str]) -> None: + nonlocal deleted + if rag is None: + return + deduplicated = list(dict.fromkeys(stale_ids)) + for start in range(0, len(deduplicated), args.batch_size): + deleted += int( + rag.delete_documents( + deduplicated[start : start + args.batch_size], + namespace=args.namespace, + ) + ) - raw_skip = 0 - sort = [("_id", 1)] - - while raw_skip < raw_to_scan: - docs = collect_text_relation_docs_for_namespace( - namespace=args.namespace, - predicates=args.predicates, - languages=args.languages, - concept_ids=args.concept_ids, - updated_since=args.updated_since, - skip=raw_skip, - sort=sort, - limit=min(args.page_size, raw_to_scan - raw_skip), - ) - - total_candidates += len(docs) - - if rag is not None and docs: - for batch in _chunks(docs, args.batch_size): + source = iter_text_relation_docs_for_namespace( + namespace=args.namespace, + predicates=args.predicates, + languages=args.languages, + concept_ids=args.concept_ids, + updated_since=args.updated_since, + limit=args.scan_limit or None, + source_batch_size=args.page_size, + stale_doc_sink=_delete_stale_batch if rag is not None else None, + ) + pending_batch: List[TextRelationRagDoc] = [] + progress_interval = max(args.page_size * 2, 1) + for doc in source: + total_candidates += 1 + if rag is not None: + pending_batch.append(doc) + if len(pending_batch) >= args.batch_size: payload = [ - {"id": d.doc_id, "text": d.text, "metadata": d.metadata} - for d in batch + {"id": item.doc_id, "text": item.text, "metadata": item.metadata} + for item in pending_batch ] - ok, bad = rag.upsert_documents(payload, namespace=args.namespace) + ok, bad = rag.upsert_documents( + payload, + namespace=args.namespace, + ) added += int(ok) failed += int(bad) - - raw_skip += args.page_size - - if raw_skip % max(args.page_size * 2, 1) == 0: + pending_batch = [] + if total_candidates % progress_interval == 0: logger.info( - "Progress: scanned=%s/%s candidates=%s added=%s failed=%s", - min(raw_skip, raw_to_scan), - raw_to_scan, + "Progress: scanned=%s scan_limit=%s candidates=%s added=%s failed=%s", + total_candidates, + args.scan_limit or "unbounded", total_candidates, added, failed, ) + if rag is not None and pending_batch: + payload = [ + {"id": item.doc_id, "text": item.text, "metadata": item.metadata} + for item in pending_batch + ] + ok, bad = rag.upsert_documents(payload, namespace=args.namespace) + added += int(ok) + failed += int(bad) + success = failed == 0 logger.info( - "Finished reindex: success=%s candidates=%s added=%s failed=%s", + "Finished reindex: success=%s candidates=%s added=%s failed=%s " + "pruned=%s", success, total_candidates, added, failed, + deleted, ) if args.dry_run: diff --git a/tests/backend/test_adaptive_turn_service.py b/tests/backend/test_adaptive_turn_service.py index 865ced0d1..efedc73ff 100644 --- a/tests/backend/test_adaptive_turn_service.py +++ b/tests/backend/test_adaptive_turn_service.py @@ -35,6 +35,7 @@ _compact_context_after_limit, _compact_evidence_index, _effect_subject_authorised, + _effect_subject_authority_denial, _effect_result_target_ids, _final_synthesis_context, _json_bytes, @@ -249,6 +250,7 @@ def _effect_gateway( write_timeout_sec: float = 1.0, effect_output_schema: Schema | None = None, effect_admission_window_sec: float | None = None, + include_scoped_assertion: bool = False, ) -> InternalMCPGateway: catalogue = MethodCatalogue() catalogue.register( @@ -263,6 +265,11 @@ def _effect_gateway( ("create_concepts", None), ("upsert_text_relation", "concept_id"), ("add_relationship", "source_id"), + *( + (("upsert_scoped_assertion", None),) + if include_scoped_assertion + else () + ), ): catalogue.register( MethodDefinition( @@ -283,7 +290,19 @@ def _effect_gateway( else ( {"namespace": "turn_namespace"} if name == "upsert_text_relation" - else None + else ( + { + "acting_user_concept_id": ( + "actor_user_concept_id" + ), + "organisation_concept_id": ( + "actor_organisation_concept_id" + ), + "namespace": "turn_namespace", + } + if name == "upsert_scoped_assertion" + else None + ) ) ), ordinary_turn_fixed_arguments=( @@ -297,7 +316,11 @@ def _effect_gateway( else ( {"provenance": None} if name == "upsert_text_relation" - else None + else ( + {"canonical_publication": False} + if name == "upsert_scoped_assertion" + else None + ) ) ), ) @@ -2039,6 +2062,312 @@ def handler(name: str, _arguments: dict[str, Any]) -> dict[str, Any]: ) +def test_canonical_text_denial_exposes_scoped_assertion_recovery( + monkeypatch, +) -> None: + from src.backend.db import mongo_client + + class _Collection: + def find_one(self, *_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"relationships": {}} + + monkeypatch.setattr( + mongo_client, + "get_concepts_collection", + lambda: _Collection(), + ) + invoked: list[tuple[str, dict[str, Any]]] = [] + + def handler(name: str, arguments: dict[str, Any]) -> dict[str, Any]: + invoked.append((name, arguments)) + return { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_recovered", + "canonical_publication": False, + } + + client = _SequenceClient( + LLMResponse( + text_response="", + tool_calls=[ + ToolCall( + tool_name="turn_invoke_capability", + call_id="canonical-text-denied", + payload={ + "name": "upsert_text_relation", + "arguments": { + "concept_id": "#V#globally_visible_subject", + "predicate": "hasNote", + "text": "Actor-relative observation.", + "language": "en-NZ", + }, + }, + ) + ], + ), + LLMResponse( + text_response="", + tool_calls=[ + ToolCall( + tool_name="turn_invoke_capability", + call_id="scoped-text-recovery", + payload={ + "name": "upsert_scoped_assertion", + "arguments": { + "subject_concept_id": ( + "#V#globally_visible_subject" + ), + "predicate": "hasNote", + "target_text": "Actor-relative observation.", + "language": "en-NZ", + }, + }, + ) + ], + ), + LLMResponse( + text_response="The actor-scoped assertion was recorded." + ), + ) + + result = execute_adaptive_turn( + gateway=_effect_gateway( + handler, + include_scoped_assertion=True, + ), + prompt="Record this observation without changing shared publication.", + context=[], + llm_client=client, + model="test-model", + user_namespace="#V#person@org", + user_concept_id="#V#person", + org_concept_id="#V#org", + turn_id="canonical-text-denied", + turn_budget_seconds=10, + final_synthesis_reserve_seconds=2, + ) + + assert len(invoked) == 1 + recovered_name, recovered_arguments = invoked[0] + assert recovered_name == "upsert_scoped_assertion" + assert recovered_arguments["acting_user_concept_id"] == "#V#person" + assert recovered_arguments["organisation_concept_id"] == "#V#org" + assert recovered_arguments["namespace"] == "#V#person@org" + assert recovered_arguments["canonical_publication"] is False + + denial = result.tool_invocations[0] + assert denial["effect_status"] == "failed" + preview = denial["evidence"]["preview"] + assert "effect_subject_not_authorised" in preview + assert "assert_in_actor_scope" in preview + assert "upsert_scoped_assertion" in preview + assert "#V#globally_visible_subject" in preview + assert "Actor-relative observation." in preview + recovery = result.tool_invocations[1] + assert recovery["effect_status"] == "succeeded" + assert recovery["changed"] is True + assert result.terminal_status == "completed" + assert result.response_text == "The actor-scoped assertion was recorded." + assert denial["recovery_status"] == "succeeded" + assert denial["recovered_by_effect_id"] == recovery["effect_id"] + + +@pytest.mark.parametrize( + "predicate_arguments", + [ + {"predicate": "#V#hasResearchInterest"}, + { + "predicate_ref": { + "concept_id": "#V#hasResearchInterest", + "value_kind": "concept", + } + }, + ], +) +def test_canonical_relationship_denial_recovers_as_scoped_assertion( + monkeypatch: pytest.MonkeyPatch, + predicate_arguments: dict[str, Any], +) -> None: + from src.backend.db import mongo_client + + class _Collection: + def find_one(self, *_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"relationships": {}} + + monkeypatch.setattr( + mongo_client, + "get_concepts_collection", + lambda: _Collection(), + ) + invoked: list[tuple[str, dict[str, Any]]] = [] + + def handler(name: str, arguments: dict[str, Any]) -> dict[str, Any]: + invoked.append((name, arguments)) + return { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_relationship_recovered", + "canonical_publication": False, + } + + client = _SequenceClient( + LLMResponse( + text_response="", + tool_calls=[ + ToolCall( + tool_name="turn_invoke_capability", + call_id="canonical-relationship-denied", + payload={ + "name": "add_relationship", + "arguments": { + "source_id": "#V#globally_visible_subject", + **predicate_arguments, + "target": "#V#represented_topic", + }, + }, + ) + ], + ), + LLMResponse( + text_response="", + tool_calls=[ + ToolCall( + tool_name="turn_invoke_capability", + call_id="scoped-relationship-recovery", + payload={ + "name": "upsert_scoped_assertion", + "arguments": { + "subject_concept_id": ( + "#V#globally_visible_subject" + ), + "predicate": "#V#hasResearchInterest", + "target_concept_id": "#V#represented_topic", + }, + }, + ) + ], + ), + LLMResponse( + text_response="The actor-scoped relationship was recorded." + ), + ) + + result = execute_adaptive_turn( + gateway=_effect_gateway( + handler, + include_scoped_assertion=True, + ), + prompt=( + "Record this represented relationship without changing shared " + "publication." + ), + context=[], + llm_client=client, + model="test-model", + user_namespace="#V#person@org", + user_concept_id="#V#person", + org_concept_id="#V#org", + turn_id="canonical-relationship-denied", + turn_budget_seconds=10, + final_synthesis_reserve_seconds=2, + ) + + assert len(invoked) == 1 + recovered_name, recovered_arguments = invoked[0] + assert recovered_name == "upsert_scoped_assertion" + assert recovered_arguments["subject_concept_id"] == ( + "#V#globally_visible_subject" + ) + assert recovered_arguments["predicate"] == "#V#hasResearchInterest" + assert recovered_arguments["target_concept_id"] == "#V#represented_topic" + assert recovered_arguments["acting_user_concept_id"] == "#V#person" + assert recovered_arguments["organisation_concept_id"] == "#V#org" + assert recovered_arguments["namespace"] == "#V#person@org" + assert recovered_arguments["canonical_publication"] is False + + denial = result.tool_invocations[0] + assert denial["effect_status"] == "failed" + preview = denial["evidence"]["preview"] + assert "effect_subject_not_authorised" in preview + assert "assert_in_actor_scope" in preview + assert "#V#globally_visible_subject" in preview + assert "#V#hasResearchInterest" in preview + assert "#V#represented_topic" in preview + recovery = result.tool_invocations[1] + assert recovery["effect_status"] == "succeeded" + assert recovery["changed"] is True + assert result.terminal_status == "completed" + assert result.response_text == ( + "The actor-scoped relationship was recorded." + ) + assert denial["recovery_status"] == "succeeded" + assert denial["recovered_by_effect_id"] == recovery["effect_id"] + + +@pytest.mark.parametrize( + "arguments", + [ + { + "source_id": "#V#subject", + "predicate": "hasResearchInterest", + "target": "#V#topic", + }, + { + "source_id": "#V#subject", + "predicate": "#V#hasResearchInterest", + "target": "Natural-language target", + }, + { + "source_id": "#V#subject", + "predicate_ref": {"name": "hasResearchInterest"}, + "target": "#V#topic", + }, + { + "source_id": "#V#subject", + "predicate": "#V#hasResearchInterest", + "predicate_ref": {"concept_id": "#V#otherPredicate"}, + "target": "#V#topic", + }, + { + "source_id": "#V#subject", + "predicate_ref": { + "concept_id": "#V#hasResearchInterest", + "value_kind": "text", + }, + "target": "#V#topic", + }, + { + "source_id": "#V#subject", + "predicate_ref": { + "concept_id": "#V#hasResearchInterest", + "on_missing": "create_typed_predicate", + }, + "target": "#V#topic", + }, + { + "source_id": "#V#subject", + "predicate": "#V#hasResearchInterest", + "predicate_if_missing": {"name": "hasResearchInterest"}, + "target": "#V#topic", + }, + ], +) +def test_non_exact_relationship_denial_has_no_scoped_recovery( + arguments: dict[str, Any], +) -> None: + denial = _effect_subject_authority_denial( + capability_name="add_relationship", + arguments=arguments, + scoped_assertion_available=True, + ) + + assert denial["error_code"] == "effect_subject_not_authorised" + assert "recovery_affordances" not in denial + + @pytest.mark.parametrize( "predicate", [ diff --git a/tests/backend/test_context_semantic_coherence.py b/tests/backend/test_context_semantic_coherence.py new file mode 100644 index 000000000..fa462cc3c --- /dev/null +++ b/tests/backend/test_context_semantic_coherence.py @@ -0,0 +1,853 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +import json + +import mongomock +from bson import ObjectId + + +def test_actor_effective_text_is_not_base_publication(monkeypatch): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service + from src.backend.services import testing_theory_service + from src.backend.services import text_value_service + + relation_id = ObjectId() + text_value_id = ObjectId() + collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + monkeypatch.setattr( + scoped_assertion_service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + scoped_assertion_service, + "can_access_concept", + lambda _concept_id: True, + ) + monkeypatch.setattr( + text_value_service.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [ + { + "_id": relation_id, + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "object_text_id": text_value_id, + } + ], + ) + monkeypatch.setattr( + text_value_service.TextValuesRepository, + "find", + lambda *_args, **_kwargs: [ + { + "_id": text_value_id, + "text": "Base programme context.", + "lang": "en-NZ", + } + ], + ) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + monkeypatch.setattr( + testing_theory_service, + "get_testing_theory_state", + lambda _theory_id: { + "local_assertions": [ + { + "assertion_id": "theory_assert_1", + "source_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "target": "Scoped programme context.", + "target_kind": "text", + "status": "proposed", + }, + { + "assertion_id": "theory_assert_2", + "source_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "target": "Base programme context.", + "target_kind": "text", + "status": "proposed", + }, + ] + }, + ) + + scoped_assertion_service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Scoped programme context.", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + + with override_current_actor("#V#michael_witbrock", "#V#trusted_org"): + effective_rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + limit=1, + context_view="actor_effective", + ) + base_rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + limit=1, + context_view="base_publication", + ) + default_rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + limit=1, + ) + diff_result = testing_theory_service.compute_testing_theory_diff( + theory_id="#V#test_theory", + ) + + assert [row["text"] for row in effective_rows] == [ + "Scoped programme context." + ] + assert effective_rows[0]["row_kind"] == "scoped_assertion" + assert effective_rows[0]["relation_id"] is None + assert effective_rows[0]["assertion_id"].startswith("ska_") + assert [row["text"] for row in base_rows] == ["Base programme context."] + assert base_rows[0]["row_kind"] == "base_text_relation" + assert default_rows == base_rows + assert diff_result["success"] is True + diff = diff_result["diff"] + assert diff["promotion_ready_assertion_ids"] == ["theory_assert_1"] + assert diff["already_canonical_assertion_ids"] == ["theory_assert_2"] + assert diff["assertions"][0]["diff_status"] == "promotion_ready" + assert diff["assertions"][1]["diff_status"] == "already_canonical" + + +def test_actor_effective_language_filter_precedes_scoped_limit(monkeypatch): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service + from src.backend.services import text_value_service + + collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + monkeypatch.setattr( + scoped_assertion_service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + scoped_assertion_service, + "can_access_concept", + lambda _concept_id: True, + ) + monkeypatch.setattr( + text_value_service.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + + en_receipt = scoped_assertion_service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="English programme context.", + language="en-NZ", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + fr_receipt = scoped_assertion_service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Contexte du programme.", + language="fr", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + collection.update_one( + {"assertion_id": en_receipt["assertion_id"]}, + {"$set": {"updated_at": datetime(2026, 7, 29, 10, 0, tzinfo=timezone.utc)}}, + ) + collection.update_one( + {"assertion_id": fr_receipt["assertion_id"]}, + {"$set": {"updated_at": datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc)}}, + ) + + with override_current_actor("#V#michael_witbrock", "#V#trusted_org"): + rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + lang="en-NZ", + limit=1, + context_view="actor_effective", + ) + + assert [row["text"] for row in rows] == ["English programme context."] + + +def test_actor_effective_multi_concept_read_uses_per_subject_pages(monkeypatch): + from src.backend.services import scoped_assertion_service + from src.backend.services import text_value_service + + concept_ids = ["#V#assertion_rich", "#V#otherwise_starved"] + observed: list[dict[str, object]] = [] + monkeypatch.setattr( + text_value_service.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda candidate_ids: set(candidate_ids), + ) + + def _page(**kwargs): + observed.append(dict(kwargs)) + items = [ + { + "assertion_id": f"ska_{index}", + "subject_concept_id": concept_id, + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": f"Scoped text {index}", + "language": "en-NZ", + }, + "canonical_publication": False, + } + for index, concept_id in enumerate(concept_ids) + ] + return { + "items": items, + "returned": 2, + "truncated": False, + "visibility_filtered": False, + "counts_are_lower_bounds": False, + "per_subject": { + concept_id: { + "returned": 1, + "limit": 1, + "has_more": False, + "visibility_filtered": False, + "counts_are_lower_bounds": False, + } + for concept_id in concept_ids + }, + } + + monkeypatch.setattr( + scoped_assertion_service, + "list_visible_scoped_assertions_page", + _page, + ) + query_metadata: dict[str, object] = {} + + rows = text_value_service.get_texts_for_concepts( + concept_ids, + predicate="hasDescription", + limit_per_concept=1, + context_view="actor_effective", + query_metadata=query_metadata, + ) + + assert [row["text"] for row in rows[concept_ids[0]]] == ["Scoped text 0"] + assert [row["text"] for row in rows[concept_ids[1]]] == ["Scoped text 1"] + assert observed[0]["subject_concept_ids"] == concept_ids + assert observed[0]["limit_per_subject"] == 1 + assert query_metadata["scoped_query_truncated"] is False + + +def test_actor_effective_summary_keeps_relation_and_assertion_ids_distinct( + monkeypatch, +): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service + from src.backend.services import text_value_service + + relation_id = ObjectId() + text_value_id = ObjectId() + base_updated_at = datetime(2026, 7, 29, 11, 0, tzinfo=timezone.utc) + scoped_updated_at = "2026-07-29T12:00:00+02:00" + collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + monkeypatch.setattr( + scoped_assertion_service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + scoped_assertion_service, + "can_access_concept", + lambda _concept_id: True, + ) + monkeypatch.setattr( + text_value_service, + "can_access_concept", + lambda _concept_id: True, + ) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + monkeypatch.setattr( + text_value_service.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [ + { + "_id": relation_id, + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "object_text_id": text_value_id, + "updated_at": base_updated_at, + } + ], + ) + monkeypatch.setattr( + text_value_service.TextValuesRepository, + "find", + lambda *_args, **_kwargs: [ + { + "_id": text_value_id, + "lang": "en-NZ", + } + ], + ) + receipt = scoped_assertion_service.upsert_scoped_assertion( + subject_concept_id="#V#gillian_dobbie", + predicate="hasDescription", + target_text="Scoped programme context.", + acting_user_concept_id="#V#michael_witbrock", + organisation_concept_id="#V#trusted_org", + namespace="#V#michael_witbrock@trusted_org", + canonical_publication=False, + ) + collection.update_one( + {"assertion_id": receipt["assertion_id"]}, + {"$set": {"updated_at": scoped_updated_at}}, + ) + + with override_current_actor("#V#michael_witbrock", "#V#trusted_org"): + summary = text_value_service.get_text_relations_summary( + "#V#gillian_dobbie", + predicates=["hasDescription"], + context_view="actor_effective", + ) + recent_rows = text_value_service.get_texts_for_concept( + "#V#gillian_dobbie", + predicate="hasDescription", + limit=2, + recent_first=True, + context_view="actor_effective", + ) + + assert summary["context_view"] == "actor_effective" + assert summary["scoped_query_truncated"] is False + assert summary["counts_are_lower_bounds"] is False + assert summary["groups_found"] == 1 + group = summary["groups"][0] + assert group["relation_ids"] == [str(relation_id)] + assert group["assertion_ids"] == [receipt["assertion_id"]] + assert group["latest_row_kind"] == "base_text_relation" + assert group["latest_relation_id"] == str(relation_id) + assert group["latest_assertion_id"] is None + assert group["latest_updated_at"] == base_updated_at.isoformat() + assert [row["row_kind"] for row in recent_rows] == [ + "base_text_relation", + "scoped_assertion", + ] + + +def test_scoped_concept_hit_does_not_masquerade_as_relation_id(monkeypatch): + from src.backend.services import concept_relation_service + from src.backend.services import scoped_assertion_service + + monkeypatch.setattr( + concept_relation_service, + "_load_accessible_relation_subject_document", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + concept_relation_service, + "query_relationship_extent_index", + lambda **_kwargs: ([], 0), + ) + monkeypatch.setattr( + concept_relation_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + monkeypatch.setattr( + scoped_assertion_service, + "list_visible_scoped_assertions_page", + lambda **_kwargs: { + "items": [ + { + "assertion_id": "ska_123", + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "#V#co_directs", + "object_kind": "concept", + "object_concept_id": "#V#research_programme", + "scope": {"mode": "user"}, + "provenance": {"turn_id": "turn-1"}, + }, + { + "assertion_id": "ska_456", + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "#V#co_directs", + "object_kind": "concept", + "object_concept_id": "#V#research_programme", + "scope": {"mode": "organisation"}, + "provenance": {"turn_id": "turn-2"}, + }, + ], + "has_more": False, + "counts_are_lower_bounds": False, + }, + ) + + base_payload = concept_relation_service.find_relations_with_argument( + "#V#gillian_dobbie", + predicate_filter=["#V#co_directs"], + relation_kind="binary", + include_concept_preview=False, + ) + payload = concept_relation_service.find_relations_with_argument( + "#V#gillian_dobbie", + predicate_filter=["#V#co_directs"], + relation_kind="binary", + include_concept_preview=False, + context_view="actor_effective", + ) + + assert base_payload["context_view"] == "base_publication" + assert base_payload["total_hits"] == 0 + assert payload["context_view"] == "actor_effective" + assert payload["total_hits"] == 2 + metadata = [hit["relation_metadata"] for hit in payload["hits"]] + assert {item["assertion_id"] for item in metadata} == {"ska_123", "ska_456"} + assert all(item["relation_id"] is None for item in metadata) + assert all(item["row_kind"] == "scoped_assertion" for item in metadata) + + +def test_relation_lookup_uses_actor_effective_text_without_fabricated_id( + monkeypatch, +): + from src.backend.services import concept_relation_service + + observed: list[dict[str, object]] = [] + + def _get_texts_for_concepts(concept_ids, **kwargs): + observed.append({"concept_ids": list(concept_ids), **kwargs}) + return { + "#V#gillian_dobbie": [ + { + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "text": "Scoped programme context.", + "lang": "en-NZ", + "relation_id": None, + "assertion_id": "ska_123", + "row_kind": "scoped_assertion", + "canonical_publication": False, + "storage_surface": "scoped_knowledge_assertions", + } + ] + } + + monkeypatch.setattr( + concept_relation_service, + "_load_accessible_relation_subject_document", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + concept_relation_service, + "get_texts_for_concepts", + _get_texts_for_concepts, + ) + + payload = concept_relation_service.find_relations_with_argument( + "#V#gillian_dobbie", + argument_index="subject", + relation_kind="text", + include_concept_preview=False, + context_view="actor_effective", + ) + + assert observed == [ + { + "concept_ids": ["#V#gillian_dobbie"], + "limit_per_concept": 501, + "context_view": "actor_effective", + "query_metadata": {}, + } + ] + assert payload["total_hits"] == 1 + assert payload.get("total_hits_is_lower_bound", False) is False + metadata = payload["hits"][0]["relation_metadata"] + assert metadata["relation_id"] is None + assert metadata["assertion_id"] == "ska_123" + assert metadata["row_kind"] == "scoped_assertion" + + +def test_relation_lookup_propagates_scoped_page_completeness(monkeypatch): + from src.backend.services import concept_relation_service + from src.backend.services import scoped_assertion_service + + monkeypatch.setattr( + concept_relation_service, + "_load_accessible_relation_subject_document", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + concept_relation_service, + "query_relationship_extent_index", + lambda **_kwargs: ([], 0), + ) + monkeypatch.setattr( + concept_relation_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + monkeypatch.setattr( + scoped_assertion_service, + "list_visible_scoped_assertions_page", + lambda **_kwargs: { + "items": [], + "has_more": True, + "counts_are_lower_bounds": True, + }, + ) + + payload = concept_relation_service.find_relations_with_argument( + "#V#subject", + relation_kind="binary", + include_concept_preview=False, + context_view="actor_effective", + ) + + assert payload["total_hits_is_lower_bound"] is True + assert ( + payload["relation_query_diagnostics"][ + "scoped_concept_query_truncated" + ] + is True + ) + + +def test_relation_lookup_propagates_actor_text_query_completeness( + monkeypatch, +): + from src.backend.services import concept_relation_service + + monkeypatch.setattr( + concept_relation_service, + "_load_accessible_relation_subject_document", + lambda *_args, **_kwargs: None, + ) + + def _get_texts(concept_ids, **kwargs): + kwargs["query_metadata"]["scoped_counts_are_lower_bounds"] = True + return {concept_id: [] for concept_id in concept_ids} + + monkeypatch.setattr( + concept_relation_service, + "get_texts_for_concepts", + _get_texts, + ) + + payload = concept_relation_service.find_relations_with_argument( + "#V#subject", + argument_index="subject", + relation_kind="text", + include_concept_preview=False, + context_view="actor_effective", + ) + + assert payload["total_hits_is_lower_bound"] is True + assert ( + payload["relation_query_diagnostics"][ + "actor_effective_text_query_truncated" + ] + is True + ) + + +def test_actor_facing_text_tools_select_the_effective_view(monkeypatch): + from src.backend.integrations.internal_mcp import catalogue + from src.backend.security.access_control import override_current_actor + from src.backend.services import text_value_service + + observed: list[tuple[str, str]] = [] + + def _get_texts_for_concept(**kwargs): + observed.append(("relations", kwargs["context_view"])) + return [] + + def _get_text_relations_summary(_concept_id, **kwargs): + observed.append(("summary", kwargs["context_view"])) + return { + "success": True, + "concept_id": "#V#gillian_dobbie", + "groups": [], + "groups_found": 0, + "total_relations_scanned": 0, + "max_relation_ids_per_group": 25, + } + + monkeypatch.setattr( + text_value_service, + "get_texts_for_concept", + _get_texts_for_concept, + ) + monkeypatch.setattr( + text_value_service, + "get_text_relations_summary", + _get_text_relations_summary, + ) + + with override_current_actor("#V#trusted_user", "#V#trusted_org"): + catalogue._get_text_relations(concept_id="#V#gillian_dobbie") + catalogue._get_text_relations_summary(concept_id="#V#gillian_dobbie") + + assert observed == [ + ("relations", "actor_effective"), + ("summary", "actor_effective"), + ] + + +def test_anonymous_stdio_text_tools_select_base_publication(monkeypatch): + from src.backend.mcp_server import process_guard + + monkeypatch.setattr( + process_guard, + "activate_mcp_helper_lifecycle", + lambda *_args, **_kwargs: {}, + ) + from src.backend.mcp_server import mcp_stdio_server + from src.backend.security.access_control import ( + should_enforce_access_control, + ) + + observed: list[tuple[str, str, bool]] = [] + + def _get_texts_for_concept(**kwargs): + observed.append( + ( + "relations", + kwargs["context_view"], + should_enforce_access_control(), + ) + ) + return [] + + def _get_text_relations_summary(_concept_id, **kwargs): + observed.append( + ( + "summary", + kwargs["context_view"], + should_enforce_access_control(), + ) + ) + return { + "success": True, + "concept_id": "#V#gillian_dobbie", + "context_view": kwargs["context_view"], + "groups": [], + "groups_found": 0, + "total_relations_scanned": 0, + "max_relation_ids_per_group": 25, + } + + monkeypatch.setattr( + mcp_stdio_server, + "get_texts_for_concept", + _get_texts_for_concept, + ) + monkeypatch.setattr( + mcp_stdio_server, + "get_text_relations_summary", + _get_text_relations_summary, + ) + + async def _invoke(): + relations = await mcp_stdio_server._handle_get_text_relations( + {"concept_id": "#V#gillian_dobbie"} + ) + summary = await mcp_stdio_server._handle_get_text_relations_summary( + {"concept_id": "#V#gillian_dobbie"} + ) + return json.loads(relations[0].text), json.loads(summary[0].text) + + relations_payload, summary_payload = asyncio.run(_invoke()) + + assert observed == [ + ("relations", "base_publication", True), + ("summary", "base_publication", True), + ] + assert relations_payload["context_view"] == "base_publication" + assert summary_payload["context_view"] == "base_publication" + + +def test_anonymous_stdio_concept_reads_enforce_global_visibility(monkeypatch): + from src.backend.mcp_server import process_guard + + monkeypatch.setattr( + process_guard, + "activate_mcp_helper_lifecycle", + lambda *_args, **_kwargs: {}, + ) + from src.backend.mcp_server import mcp_stdio_server + from src.backend.security.access_control import ( + should_enforce_access_control, + ) + + observed: list[tuple[str, bool]] = [] + + def _get_concept(concept_id): + observed.append(("fetch", should_enforce_access_control())) + return {"concept_id": concept_id, "relationships": {}} + + def _find_relations(**_kwargs): + observed.append(("relations", should_enforce_access_control())) + return {"success": True, "relations": []} + + monkeypatch.setattr( + mcp_stdio_server, + "get_concept_by_concept_id", + _get_concept, + ) + monkeypatch.setattr( + mcp_stdio_server, + "enrich_concept_with_text_relations", + lambda concept: concept, + ) + monkeypatch.setattr( + mcp_stdio_server, + "find_relations_with_argument", + _find_relations, + ) + + async def _invoke(): + fetched = await mcp_stdio_server._handle_fetch_concept( + {"concept_id": "#V#globally_visible"} + ) + related = await mcp_stdio_server._handle_find_relations_with_argument( + {"concept_id": "#V#globally_visible"} + ) + return json.loads(fetched[0].text), json.loads(related[0].text) + + fetched_payload, related_payload = asyncio.run(_invoke()) + + assert observed == [("fetch", True), ("relations", True)] + assert fetched_payload["concept_id"] == "#V#globally_visible" + assert related_payload["success"] is True + + +def test_stdio_rag_search_rejects_untrusted_namespace_before_backend(monkeypatch): + from src.backend.mcp_server import process_guard + + monkeypatch.setattr( + process_guard, + "activate_mcp_helper_lifecycle", + lambda *_args, **_kwargs: {}, + ) + from src.backend.mcp_server import mcp_stdio_server + from src.backend.services import rag_service + + def _unexpected_backend_access(): + raise AssertionError("an untrusted namespace must not reach RAG") + + monkeypatch.setattr( + rag_service, + "get_rag_service", + _unexpected_backend_access, + ) + + async def _invoke(): + result = await mcp_stdio_server.call_tool( + "search_knowledge_base", + { + "query": "private programme", + "namespace": "#V#victim@private_org", + "user_concept_id": "#V#victim", + "organisation_concept_id": "#V#private_org", + } + ) + return json.loads(result[0].text) + + payload = asyncio.run(_invoke()) + + assert payload["success"] is False + assert payload["error"] == "namespace_required" + assert payload["namespace_source"] == "untrusted_payload_rejected" + assert ( + payload["namespace_resolution_note"] + == "authenticated_actor_context_required" + ) + + +def test_stdio_rag_search_uses_server_bound_actor(monkeypatch): + from src.backend.mcp_server import process_guard + + monkeypatch.setattr( + process_guard, + "activate_mcp_helper_lifecycle", + lambda *_args, **_kwargs: {}, + ) + from src.backend.mcp_server import mcp_stdio_server + from src.backend.security.access_control import override_current_actor + from src.backend.services import rag_service + + observed: dict[str, object] = {} + + class _StubRAG: + def query(self, **kwargs): + observed.update(kwargs) + return [ + { + "id": "doc-1", + "text": "Authorised programme context.", + "metadata": {}, + "score": 0.9, + } + ] + + monkeypatch.setattr(rag_service, "get_rag_service", lambda: _StubRAG()) + + async def _invoke(): + result = await mcp_stdio_server.call_tool( + "search_knowledge_base", + {"query": "programme context"} + ) + return json.loads(result[0].text) + + with override_current_actor("#V#trusted_user", "#V#trusted_org"): + payload = asyncio.run(_invoke()) + + assert payload["success"] is True + assert payload["namespace"] == "#V#trusted_user@trusted_org" + assert payload["namespace_source"] == "trusted_actor_context" + assert observed["namespace"] == "#V#trusted_user@trusted_org" + assert observed["permissions_context"] == { + "user_id": "#V#trusted_user", + "organisation_concept_id": "#V#trusted_org", + } diff --git a/tests/backend/test_description_metadata_preservation.py b/tests/backend/test_description_metadata_preservation.py index 86a1f5c6a..72d905d02 100644 --- a/tests/backend/test_description_metadata_preservation.py +++ b/tests/backend/test_description_metadata_preservation.py @@ -76,12 +76,18 @@ def test_get_texts_for_concept_returns_provenance_and_timestamps(monkeypatch): now = datetime(2026, 3, 1, tzinfo=timezone.utc) monkeypatch.setattr(text_value_service, "can_access_concept", lambda *_: True) + monkeypatch.setattr( + text_value_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) monkeypatch.setattr( text_value_service.TextRelationsRepository, "find", lambda *args, **kwargs: [ { "_id": relation_id, + "subject_concept_id": "#V#desc_test", "predicate": "hasDescription", "object_text_id": text_value_id, "context": {"confidence_score": 0.66}, diff --git a/tests/backend/test_internal_mcp_catalogue_builds.py b/tests/backend/test_internal_mcp_catalogue_builds.py index 8184101b7..0527f22a3 100644 --- a/tests/backend/test_internal_mcp_catalogue_builds.py +++ b/tests/backend/test_internal_mcp_catalogue_builds.py @@ -6,6 +6,7 @@ def test_internal_mcp_catalogue_builds_and_includes_relationship_tools(): assert "add_relationship" in methods assert "upsert_scoped_assertion" in methods + assert "retract_scoped_assertion" in methods assert "list_scoped_assertions" in methods assert "remove_relationship" in methods assert "preview_remove_relationship" in methods @@ -74,6 +75,10 @@ def test_default_catalogue_exposes_calibrated_effect_admission_windows(): assert catalogue.get("create_concepts").effect_admission_window_sec is None assert catalogue.get("upsert_text_relation").effect_admission_window_sec == 8.0 + assert ( + catalogue.get("retract_scoped_assertion").effect_admission_window_sec + == 8.0 + ) assert catalogue.get("add_relationship").effect_admission_window_sec == 5.0 assert snapshot["create_concepts"]["effect_admission_window_sec"] is None assert snapshot["upsert_text_relation"]["effect_admission_window_sec"] == 8.0 @@ -147,6 +152,7 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): }.isdisjoint(actor_mail_reads) assert delegated_effects == { "create_concepts", + "retract_scoped_assertion", "upsert_scoped_assertion", "upsert_text_relation", "add_relationship", @@ -180,6 +186,31 @@ def test_ordinary_turn_read_projection_follows_capability_authority_metadata(): "canonical_publication": False, } assert scoped_definition.ordinary_turn_mutation_subject_argument is None + retract_definition = catalogue.get("retract_scoped_assertion") + assert retract_definition.input_schema.required == {"assertion_id": str} + assert retract_definition.input_schema.optional == {} + assert retract_definition.ordinary_turn_trusted_argument_bindings == { + "acting_user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + "namespace": "turn_namespace", + } + assert catalogue.get( + "search_knowledge_base" + ).ordinary_turn_trusted_argument_bindings == { + "namespace": "turn_namespace", + "user_concept_id": "actor_user_concept_id", + "organisation_concept_id": "actor_organisation_concept_id", + } + for capability_name in ( + "rag_list_collections", + "rag_list_indexed", + "rag_get_item", + ): + assert catalogue.get( + capability_name + ).ordinary_turn_trusted_argument_bindings == { + "namespace": "turn_namespace", + } # Server-bound resources are absent until the entry point supplies the # actor's represented binding; the model never selects the profile. diff --git a/tests/backend/test_internal_mcp_concept_exists_gateway.py b/tests/backend/test_internal_mcp_concept_exists_gateway.py index 78c262bd4..66fcc89dc 100644 --- a/tests/backend/test_internal_mcp_concept_exists_gateway.py +++ b/tests/backend/test_internal_mcp_concept_exists_gateway.py @@ -61,7 +61,6 @@ def test_concept_exists_binds_namespace_actor_for_access_decision( "src.backend.security.access_control.get_concepts_collection", lambda: collection, ) - allowed = gateway.invoke( "concept_exists", {"concept_id": "#V#team_only_concept", "namespace": "#V#member@sail"}, @@ -99,6 +98,14 @@ def test_fetch_concept_returns_access_denied_without_leaking_payload( "src.backend.security.access_control.get_concepts_collection", lambda: collection, ) + monkeypatch.setattr( + "src.backend.security.access_control.describe_concept_access", + lambda _concept_id: { + "exists": True, + "accessible": False, + "restriction_families_present": ["specific_to_user"], + }, + ) result = gateway.invoke( "fetch_concept", diff --git a/tests/backend/test_internal_mcp_scoped_assertion_authority.py b/tests/backend/test_internal_mcp_scoped_assertion_authority.py new file mode 100644 index 000000000..44ca45746 --- /dev/null +++ b/tests/backend/test_internal_mcp_scoped_assertion_authority.py @@ -0,0 +1,820 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +import pytest + + +TRUSTED_USER = "#V#trusted_user" +TRUSTED_ORG = "#V#trusted_org" +TRUSTED_NAMESPACE = "#V#trusted_user@trusted_org" +CLAIMED_USER = "#V#payload_user" +CLAIMED_ORG = "#V#payload_org" +CLAIMED_NAMESPACE = "#V#payload_user@payload_org" + + +def _gateway(*, trusted_operator: bool = False): + from src.backend.integrations.internal_mcp import ( + InternalMCPGateway, + InternalMCPTransport, + build_default_catalogue, + ) + + return InternalMCPGateway( + catalogue=build_default_catalogue(), + transport=InternalMCPTransport(), + enabled=True, + trusted_actor_payload_fallback=trusted_operator, + ) + + +def _route_payloads( + identity: dict[str, Any] | None = None, +) -> dict[str, dict[str, Any]]: + actor = dict(identity or {}) + return { + "upsert_scoped_assertion": { + "subject_concept_id": "#V#subject", + "predicate": "#V#related_to", + "target_concept_id": "#V#target", + **actor, + }, + "retract_scoped_assertion": { + "assertion_id": "ska_existing", + **actor, + }, + "list_scoped_assertions": actor, + "get_text_relations": {"concept_id": "#V#subject", **actor}, + "get_text_relations_summary": {"concept_id": "#V#subject", **actor}, + "fetch_concept": { + "concept_id": "#V#subject", + **actor, + }, + "find_relations_with_argument": { + "concept_id": "#V#subject", + **actor, + }, + } + + +def _install_route_fakes(monkeypatch: pytest.MonkeyPatch): + from src.backend.security import access_control + + calls: dict[str, list[dict[str, Any]]] = defaultdict(list) + + def actor() -> tuple[str | None, str | None]: + return ( + access_control.get_effective_user_concept_id(), + access_control.get_effective_organisation_concept_id(), + ) + + def fake_upsert(**kwargs): + calls["upsert"].append({**kwargs, "_actor": actor()}) + return { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_test", + "assertion": { + "assertion_id": "ska_test", + "object_kind": "concept", + "predicate": kwargs["predicate"], + }, + "canonical_publication": False, + } + + def fake_retract(**kwargs): + calls["retract"].append({**kwargs, "_actor": actor()}) + assertion = { + "assertion_id": kwargs["assertion_id"], + "object_kind": "concept", + "status": "retracted", + } + return { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": kwargs["assertion_id"], + "assertion": assertion, + "canonical_read_back": assertion, + "canonical_publication": False, + } + + def fake_list(**kwargs): + calls["list"].append({**kwargs, "_actor": actor()}) + if actor() == (None, None): + return [] + if kwargs.get("limit") == 101: + return [ + {"assertion_id": f"ska_{index:03d}"} + for index in range(101) + ] + return [{"assertion_id": "ska_visible"}] + + def fake_page(**kwargs): + if kwargs.get("offset") == 40: + calls["list"].append({**kwargs, "_actor": actor()}) + return { + "items": [ + {"assertion_id": "ska_page_1"}, + {"assertion_id": "ska_page_2"}, + ], + "returned": 2, + "limit": kwargs.get("limit"), + "offset": 40, + "has_more": True, + "next_offset": 42, + "visibility_filtered": True, + "counts_are_lower_bounds": True, + } + items = fake_list(**kwargs) + return { + "items": items, + "returned": len(items), + "limit": kwargs.get("limit"), + "offset": kwargs.get("offset", 0), + "has_more": False, + "next_offset": None, + "visibility_filtered": False, + "counts_are_lower_bounds": False, + } + + def fake_texts(**kwargs): + calls["text"].append({**kwargs, "_actor": actor()}) + return [] + + def fake_summary(concept_id, **kwargs): + calls["summary"].append( + {"concept_id": concept_id, **kwargs, "_actor": actor()} + ) + return { + "success": True, + "concept_id": concept_id, + "context_view": kwargs.get("context_view"), + "groups": [], + "groups_found": 0, + "total_relations_scanned": 0, + "max_relation_ids_per_group": kwargs[ + "max_relation_ids_per_group" + ], + } + + def fake_get_concept(*, concept_id): + calls["concept"].append({"concept_id": concept_id, "_actor": actor()}) + return {"concept_id": concept_id} + + def fake_find_relations(**kwargs): + calls["relations"].append({**kwargs, "_actor": actor()}) + return { + "concept_id": kwargs["concept_id"], + "context_view": kwargs["context_view"], + "total_hits": 0, + "hits": [], + "paging": {"limit": 0, "offset": 0}, + } + + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service.upsert_scoped_assertion", + fake_upsert, + ) + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service.retract_scoped_assertion", + fake_retract, + ) + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service.list_visible_scoped_assertions", + fake_list, + ) + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service." + "list_visible_scoped_assertions_page", + fake_page, + ) + monkeypatch.setattr( + "src.backend.services.text_value_service.get_texts_for_concept", + fake_texts, + ) + monkeypatch.setattr( + "src.backend.services.text_value_service.get_text_relations_summary", + fake_summary, + ) + monkeypatch.setattr( + "src.backend.security.access_control.describe_concept_access", + lambda _concept_id: {"exists": True, "accessible": True}, + ) + monkeypatch.setattr( + "src.backend.services.concept_service.get_concept_by_concept_id", + fake_get_concept, + ) + monkeypatch.setattr( + "src.backend.services.concept_service.enrich_concept_with_text_relations", + lambda concept: concept, + ) + monkeypatch.setattr( + "src.backend.services.relationship_write_service.detect_vacuous_typing", + lambda _concept: None, + ) + monkeypatch.setattr( + "src.backend.services.concept_relation_service.find_relations_with_argument", + fake_find_relations, + ) + return calls + + +def _assert_all_routes_called_as( + calls: dict[str, list[dict[str, Any]]], + *, + expected_actor: tuple[str, str], +) -> None: + assert calls["upsert"][-1]["_actor"] == expected_actor + assert calls["retract"][-1]["_actor"] == expected_actor + assert calls["list"][-2]["_actor"] == expected_actor + assert calls["text"][-1]["_actor"] == expected_actor + assert calls["summary"][-1]["_actor"] == expected_actor + assert calls["concept"][-1]["_actor"] == expected_actor + assert calls["relations"][-1]["_actor"] == expected_actor + + +def test_generic_gateway_payload_identity_degrades_generic_reads_to_base( + monkeypatch: pytest.MonkeyPatch, +): + calls = _install_route_fakes(monkeypatch) + gateway = _gateway() + identity = { + "user_concept_id": CLAIMED_USER, + "organisation_concept_id": CLAIMED_ORG, + "namespace": CLAIMED_NAMESPACE, + } + + payloads = _route_payloads(identity) + mutation = payloads.pop("upsert_scoped_assertion") + retraction = payloads.pop("retract_scoped_assertion") + scoped_list = payloads.pop("list_scoped_assertions") + results = { + method_name: gateway.invoke(method_name, payload).payload + for method_name, payload in payloads.items() + } + + for method_name in ( + "get_text_relations", + "get_text_relations_summary", + "fetch_concept", + "find_relations_with_argument", + ): + assert results[method_name]["context_view"] == "base_publication" + assert results["fetch_concept"]["scoped_assertions"] == [] + assert calls["list"] == [] + for route_calls in calls.values(): + for call in route_calls: + assert call["_actor"] == (None, None) + + for method_name, payload in ( + ("upsert_scoped_assertion", mutation), + ("retract_scoped_assertion", retraction), + ("list_scoped_assertions", scoped_list), + ): + result = gateway.invoke(method_name, payload).payload + assert result["success"] is False, method_name + assert ( + result["error_code"] == "authenticated_actor_context_required" + ), method_name + + +def test_actorless_generic_gateway_preserves_base_compatible_reads( + monkeypatch: pytest.MonkeyPatch, +): + calls = _install_route_fakes(monkeypatch) + gateway = _gateway() + payloads = _route_payloads() + mutation = payloads.pop("upsert_scoped_assertion") + retraction = payloads.pop("retract_scoped_assertion") + scoped_list = payloads.pop("list_scoped_assertions") + + results = { + method_name: gateway.invoke(method_name, payload).payload + for method_name, payload in payloads.items() + } + denied_mutation = gateway.invoke( + "upsert_scoped_assertion", + mutation, + ).payload + denied_list = gateway.invoke( + "list_scoped_assertions", + scoped_list, + ).payload + denied_retraction = gateway.invoke( + "retract_scoped_assertion", + retraction, + ).payload + + assert all(result.get("success") is not False for result in results.values()) + assert results["fetch_concept"]["scoped_assertions"] == [] + assert results["fetch_concept"]["scoped_assertions_truncated"] is False + assert results["fetch_concept"]["context_view"] == "base_publication" + assert results["get_text_relations"]["context_view"] == "base_publication" + assert ( + results["get_text_relations_summary"]["context_view"] + == "base_publication" + ) + assert ( + results["find_relations_with_argument"]["context_view"] + == "base_publication" + ) + assert denied_mutation["error_code"] == "authenticated_actor_context_required" + assert ( + denied_retraction["error_code"] + == "authenticated_actor_context_required" + ) + assert denied_list["error_code"] == "authenticated_actor_context_required" + for route_calls in calls.values(): + for call in route_calls: + assert call["_actor"] == (None, None) + + +def test_actorless_base_binding_keeps_global_visibility_enforcement_enabled(): + from types import SimpleNamespace + + from src.backend.integrations.internal_mcp import catalogue as cat + from src.backend.security.access_control import ( + should_enforce_access_control, + ) + + actorless_scope = SimpleNamespace( + user_concept_id=None, + organisation_concept_id=None, + ) + assert should_enforce_access_control() is False + with cat._bind_internal_mcp_scoped_assertion_actor(actorless_scope): + assert should_enforce_access_control() is True + assert should_enforce_access_control() is False + + +def test_authenticated_actor_is_bound_across_every_scoped_assertion_path( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + calls = _install_route_fakes(monkeypatch) + gateway = _gateway() + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + results = { + method_name: gateway.invoke(method_name, payload).payload + for method_name, payload in _route_payloads().items() + } + + assert results["upsert_scoped_assertion"]["success"] is True + assert results["retract_scoped_assertion"]["success"] is True + assert results["list_scoped_assertions"]["success"] is True + assert results["list_scoped_assertions"]["paging"]["has_more"] is False + assert results["get_text_relations"]["context_view"] == "actor_effective" + assert ( + results["get_text_relations_summary"]["context_view"] + == "actor_effective" + ) + assert results["find_relations_with_argument"]["total_hits"] == 0 + concept = results["fetch_concept"] + assert len(concept["scoped_assertions"]) == 100 + assert concept["scoped_assertion_count"] == 100 + assert concept["scoped_assertions_truncated"] is True + assert concept["scoped_assertion_count_is_lower_bound"] is True + + _assert_all_routes_called_as( + calls, + expected_actor=(TRUSTED_USER, TRUSTED_ORG), + ) + upsert = calls["upsert"][-1] + assert upsert["acting_user_concept_id"] == TRUSTED_USER + assert upsert["organisation_concept_id"] == TRUSTED_ORG + assert upsert["namespace"] == TRUSTED_NAMESPACE + assert upsert["canonical_publication"] is False + explicit_list = calls["list"][-2] + assert explicit_list["user_concept_id"] == TRUSTED_USER + assert explicit_list["organisation_concept_id"] == TRUSTED_ORG + assert calls["relations"][-1]["context_view"] == "actor_effective" + + +def test_scoped_list_forwards_offset_and_reports_bounded_page( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + calls = _install_route_fakes(monkeypatch) + gateway = _gateway() + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = gateway.invoke( + "list_scoped_assertions", + { + "argument_concept_id": "#V#subject", + "predicates": ["hasDescription"], + "object_kind": "text", + "languages": ["en-NZ"], + "limit": 2, + "offset": 40, + }, + ).payload + + assert result["success"] is True + assert [row["assertion_id"] for row in result["assertions"]] == [ + "ska_page_1", + "ska_page_2", + ] + assert result["assertions_found"] == 2 + assert result["assertions_found_is_lower_bound"] is True + assert result["paging"] == { + "limit": 2, + "offset": 40, + "returned": 2, + "has_more": True, + "next_offset": 42, + "visibility_filtered": True, + "counts_are_lower_bounds": True, + } + forwarded = calls["list"][-1] + assert forwarded["offset"] == 40 + assert forwarded["languages"] == ["en-NZ"] + assert forwarded["_actor"] == (TRUSTED_USER, TRUSTED_ORG) + + +def test_trusted_operator_fallback_is_bound_across_every_scoped_assertion_path( + monkeypatch: pytest.MonkeyPatch, +): + calls = _install_route_fakes(monkeypatch) + gateway = _gateway(trusted_operator=True) + identity = { + "user_concept_id": TRUSTED_USER, + "organisation_concept_id": TRUSTED_ORG, + "namespace": TRUSTED_NAMESPACE, + } + + results = { + method_name: gateway.invoke(method_name, payload).payload + for method_name, payload in _route_payloads(identity).items() + } + + assert all(result.get("success") is not False for result in results.values()) + _assert_all_routes_called_as( + calls, + expected_actor=(TRUSTED_USER, TRUSTED_ORG), + ) + assert calls["upsert"][-1]["acting_user_concept_id"] == TRUSTED_USER + assert calls["upsert"][-1]["namespace"] == TRUSTED_NAMESPACE + assert calls["retract"][-1]["acting_user_concept_id"] == TRUSTED_USER + assert calls["retract"][-1]["namespace"] == TRUSTED_NAMESPACE + + +def test_authenticated_generic_reads_ignore_conflicting_payload_identity( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + calls = _install_route_fakes(monkeypatch) + gateway = _gateway() + conflicting_identity = { + "user_concept_id": CLAIMED_USER, + "organisation_concept_id": CLAIMED_ORG, + "namespace": CLAIMED_NAMESPACE, + } + + payloads = _route_payloads(conflicting_identity) + mutation = payloads.pop("upsert_scoped_assertion") + retraction = payloads.pop("retract_scoped_assertion") + scoped_list = payloads.pop("list_scoped_assertions") + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + results = { + method_name: gateway.invoke(method_name, payload).payload + for method_name, payload in payloads.items() + } + for method_name, payload in ( + ("upsert_scoped_assertion", mutation), + ("retract_scoped_assertion", retraction), + ("list_scoped_assertions", scoped_list), + ): + result = gateway.invoke(method_name, payload).payload + assert result["success"] is False, method_name + assert result["error_code"] == "workflow_actor_scope_mismatch" + + assert all( + result["context_view"] == "actor_effective" + for result in results.values() + ) + for route_calls in calls.values(): + for call in route_calls: + assert call["_actor"] == (TRUSTED_USER, TRUSTED_ORG) + + +def test_truncated_actor_effective_relation_lookup_disables_offset_continuation( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security import access_control + from src.backend.security.access_control import override_current_actor + + captured: dict[str, Any] = {} + + def truncated_lookup(**kwargs): + captured.update(kwargs) + captured["_actor"] = ( + access_control.get_effective_user_concept_id(), + access_control.get_effective_organisation_concept_id(), + ) + return { + "concept_id": kwargs["concept_id"], + "context_view": kwargs["context_view"], + "total_hits": 500, + "total_hits_is_lower_bound": True, + "hits": [], + "paging": { + "limit": 50, + "offset": 500, + "returned": 0, + "total_available": 500, + "total_available_is_lower_bound": True, + }, + "relation_query_diagnostics": { + "scoped_concept_query_truncated": True, + }, + } + + monkeypatch.setattr( + "src.backend.services.concept_relation_service." + "find_relations_with_argument", + truncated_lookup, + ) + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = _gateway().invoke( + "find_relations_with_argument", + { + "concept_id": "#V#subject", + "predicate_filter": ["#V#related_to"], + "relation_kind": "binary", + "limit": 50, + "offset": 500, + }, + ).payload + + assert captured["context_view"] == "actor_effective" + assert captured["_actor"] == (TRUSTED_USER, TRUSTED_ORG) + assert result["total_hits_is_lower_bound"] is True + assert result["paging"]["continuation_supported"] is False + assert result["paging"]["next_offset"] is None + assert ( + result["paging"]["offset_semantics"] + == "bounded_actor_overlay_snapshot" + ) + assert result["continuation"]["can_continue_with_offset"] is False + recovery = result["continuation"]["recovery_options"][1] + assert recovery["tool"] == "list_scoped_assertions" + assert recovery["scope"] == "scoped_overlay_only" + assert recovery["arguments"] == { + "argument_concept_id": "#V#subject", + "predicates": ["#V#related_to"], + "object_kind": "concept", + "limit": 200, + "offset": 0, + } + + +def test_text_assertion_receipt_includes_derived_maintenance_schedule( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + assertion = { + "assertion_id": "ska_scheduled", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Scoped context.", "language": "en-NZ"}, + } + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service.upsert_scoped_assertion", + lambda **_kwargs: { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_scheduled", + "assertion": assertion, + "canonical_read_back": assertion, + "canonical_publication": False, + }, + ) + monkeypatch.setattr( + "src.backend.services.rag_text_relation_change_hook_service." + "maybe_sync_concept_text_relations_to_rag", + lambda **_kwargs: { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": True, + "scheduled": True, + }, + ) + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = _gateway().invoke( + "upsert_scoped_assertion", + { + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "target_text": "Scoped context.", + }, + ).payload + + assert result["effect_status"] == "succeeded" + assert result["changed"] is True + assert result["derived_maintenance"] == { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": True, + "scheduled": True, + } + + +def test_text_retraction_uses_trusted_actor_and_reports_index_refresh( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + captured: dict[str, dict[str, Any]] = {} + assertion = { + "assertion_id": "ska_retracted", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "status": "retracted", + } + + def retract(**kwargs): + captured["retract"] = kwargs + return { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_retracted", + "assertion": assertion, + "canonical_read_back": assertion, + "canonical_publication": False, + } + + def schedule(**kwargs): + captured["schedule"] = kwargs + return { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": True, + "scheduled": True, + } + + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service." + "retract_scoped_assertion", + retract, + ) + monkeypatch.setattr( + "src.backend.services.rag_text_relation_change_hook_service." + "maybe_sync_concept_text_relations_to_rag", + schedule, + ) + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = _gateway().invoke( + "retract_scoped_assertion", + {"assertion_id": "ska_retracted"}, + ).payload + + assert captured["retract"] == { + "assertion_id": "ska_retracted", + "acting_user_concept_id": TRUSTED_USER, + "organisation_concept_id": TRUSTED_ORG, + "namespace": TRUSTED_NAMESPACE, + } + assert captured["schedule"] == { + "namespace": TRUSTED_NAMESPACE, + "concept_id": "#V#subject", + "predicate": "hasDescription", + "scoped_assertion_id": "ska_retracted", + } + assert result["effect_status"] == "succeeded" + assert result["changed"] is True + assert result["derived_maintenance"] == { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": True, + "scheduled": True, + } + + +def test_retraction_schedule_failure_does_not_hide_durable_receipt( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + assertion = { + "assertion_id": "ska_retracted", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "status": "retracted", + } + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service." + "retract_scoped_assertion", + lambda **_kwargs: { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_retracted", + "assertion": assertion, + "canonical_read_back": assertion, + "canonical_publication": False, + }, + ) + + def fail_schedule(**_kwargs): + raise RuntimeError("refresh queue unavailable") + + monkeypatch.setattr( + "src.backend.services.rag_text_relation_change_hook_service." + "maybe_sync_concept_text_relations_to_rag", + fail_schedule, + ) + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = _gateway().invoke( + "retract_scoped_assertion", + {"assertion_id": "ska_retracted"}, + ).payload + + assert result["effect_status"] == "succeeded" + assert result["changed"] is True + assert result["derived_maintenance"] == { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "schedule_failed", + "error": "refresh queue unavailable", + } + + +def test_rag_scheduling_failure_does_not_hide_durable_assertion_receipt( + monkeypatch: pytest.MonkeyPatch, +): + from src.backend.security.access_control import override_current_actor + + assertion = { + "assertion_id": "ska_durable", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Scoped context.", "language": "en-NZ"}, + } + monkeypatch.setattr( + "src.backend.services.scoped_assertion_service.upsert_scoped_assertion", + lambda **_kwargs: { + "success": True, + "effect_status": "succeeded", + "changed": True, + "assertion_id": "ska_durable", + "assertion": assertion, + "canonical_read_back": assertion, + "canonical_publication": False, + "storage_surface": "scoped_knowledge_assertions", + }, + ) + + def _fail_rag_schedule(**_kwargs): + raise RuntimeError("derived index unavailable") + + monkeypatch.setattr( + "src.backend.services.rag_text_relation_change_hook_service." + "maybe_sync_concept_text_relations_to_rag", + _fail_rag_schedule, + ) + + with override_current_actor(TRUSTED_USER, TRUSTED_ORG): + result = _gateway().invoke( + "upsert_scoped_assertion", + { + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "target_text": "Scoped context.", + }, + ).payload + + assert result["success"] is True + assert result["effect_status"] == "succeeded" + assert result["assertion_id"] == "ska_durable" + assert result["canonical_read_back"] == assertion + assert result["derived_maintenance"] == { + "mechanism": "in_memory_bounded_queue", + "durable": False, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "schedule_failed", + "error": "derived index unavailable", + } diff --git a/tests/backend/test_mongo_collection_index_guards.py b/tests/backend/test_mongo_collection_index_guards.py index 230d650fa..f6475e5c1 100644 --- a/tests/backend/test_mongo_collection_index_guards.py +++ b/tests/backend/test_mongo_collection_index_guards.py @@ -87,15 +87,71 @@ def test_scoped_assertion_indexes_are_ensured_once_per_database(monkeypatch): assert coll_first is coll_second assert coll_first is not None - assert len(coll_first.create_index_calls) == 4 - assert { - str(call["kwargs"].get("name")) for call in coll_first.create_index_calls - } == { + assert len(coll_first.create_index_calls) == 8 + calls_by_name = { + str(call["kwargs"].get("name")): call + for call in coll_first.create_index_calls + } + assert set(calls_by_name) == { "assertion_id_unique", - "subject_audience_updated_at_desc", - "object_audience_updated_at_desc", - "predicate_audience_updated_at_desc", + "audience_status_updated_at_desc_assertion_id", + "subject_audience_status_updated_at_desc_assertion_id", + "object_audience_status_updated_at_desc_assertion_id", + "predicate_audience_status_updated_at_desc_assertion_id", + "audience_status_id", + "subject_audience_status_id", + "predicate_audience_status_id", } + assert calls_by_name["audience_status_updated_at_desc_assertion_id"]["keys"] == [ + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("updated_at", mc.DESCENDING), + ("assertion_id", mc.ASCENDING), + ] + assert calls_by_name[ + "object_audience_status_updated_at_desc_assertion_id" + ]["keys"] == [ + ("object_concept_id", mc.ASCENDING), + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("updated_at", mc.DESCENDING), + ("assertion_id", mc.ASCENDING), + ] + assert calls_by_name[ + "predicate_audience_status_updated_at_desc_assertion_id" + ]["keys"] == [ + ("predicate", mc.ASCENDING), + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("updated_at", mc.DESCENDING), + ("assertion_id", mc.ASCENDING), + ] + assert calls_by_name[ + "subject_audience_status_updated_at_desc_assertion_id" + ]["keys"] == [ + ("subject_concept_id", mc.ASCENDING), + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("updated_at", mc.DESCENDING), + ("assertion_id", mc.ASCENDING), + ] + assert calls_by_name["audience_status_id"]["keys"] == [ + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] + assert calls_by_name["subject_audience_status_id"]["keys"] == [ + ("subject_concept_id", mc.ASCENDING), + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] + assert calls_by_name["predicate_audience_status_id"]["keys"] == [ + ("predicate", mc.ASCENDING), + ("scope.audience_keys", mc.ASCENDING), + ("status", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] def test_interaction_session_indexes_are_ensured_once_per_database(monkeypatch): @@ -210,12 +266,27 @@ def test_text_relation_indexes_cover_concept_search_and_atlas_name_lookup(monkey assert calls_by_name.keys() >= { "context_name_type_predicate_text", "object_predicate_subject_lookup", + "subject_concept_id_id", + "predicate_id", + "updated_at_id", } assert calls_by_name["object_predicate_subject_lookup"]["keys"] == [ ("object_text_id", mc.ASCENDING), ("predicate", mc.ASCENDING), ("subject_concept_id", mc.ASCENDING), ] + assert calls_by_name["subject_concept_id_id"]["keys"] == [ + ("subject_concept_id", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] + assert calls_by_name["predicate_id"]["keys"] == [ + ("predicate", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] + assert calls_by_name["updated_at_id"]["keys"] == [ + ("updated_at", mc.ASCENDING), + ("_id", mc.ASCENDING), + ] def test_relationship_extent_indexes_add_target_first_lookup_without_dropping_drift(): diff --git a/tests/backend/test_rag_provenance_contract.py b/tests/backend/test_rag_provenance_contract.py index 1a760f43f..72e521c03 100644 --- a/tests/backend/test_rag_provenance_contract.py +++ b/tests/backend/test_rag_provenance_contract.py @@ -177,6 +177,150 @@ def test_search_knowledge_base_derives_permissions_context_from_namespace(monkey } +def test_search_knowledge_base_rejects_conflicting_flask_actor(monkeypatch): + from flask import Flask, session + + from src.backend.integrations.internal_mcp import catalogue as cat + + def _unexpected_backend_access(*_args, **_kwargs): + raise AssertionError("RAG backend must not be accessed") + + monkeypatch.setattr( + "src.backend.services.rag_service.get_rag_service", + _unexpected_backend_access, + ) + app = Flask(__name__) + app.secret_key = "test-only" + + with app.test_request_context("/"): + session["user_concept_id"] = "#V#authenticated_user" + session["organisation_concept_id"] = "#V#authenticated_org" + result = cat._search_knowledge_base( + query="workflow", + namespace="#V#other_user@other_org", + ) + + assert result["success"] is False + assert result["error_code"] == "namespace_mismatch" + assert result["error_details"]["mismatch_fields"] == ["user_id"] + + +def test_default_gateway_rejects_untrusted_rag_namespace_before_backend_access( + monkeypatch, +): + from src.backend.integrations.internal_mcp import ( + InternalMCPGateway, + InternalMCPTransport, + build_default_catalogue, + ) + + def _unexpected_backend_access(*_args, **_kwargs): + raise AssertionError("RAG or database backend must not be accessed") + + monkeypatch.setattr( + "src.backend.services.rag_service.get_rag_service", + _unexpected_backend_access, + ) + monkeypatch.setattr( + "src.backend.db.connection_manager.get_db", + _unexpected_backend_access, + ) + gateway = InternalMCPGateway( + catalogue=build_default_catalogue(), + transport=InternalMCPTransport(), + enabled=True, + ) + claimed_namespace = "#V#claimed_user@claimed_org" + + calls = { + "search_knowledge_base": { + "query": "private programme", + "namespace": claimed_namespace, + }, + "rag_list_collections": {"namespace": claimed_namespace}, + "rag_list_indexed": {"namespace": claimed_namespace}, + "rag_get_item": { + "namespace": claimed_namespace, + "session_id": "claimed-session", + }, + } + for method_name, payload in calls.items(): + result = gateway.invoke(method_name, payload).payload + assert result["success"] is False + assert result["error"] == "namespace_required" + assert result["namespace_source"] == "untrusted_payload_rejected" + assert ( + result["namespace_resolution_note"] + == "authenticated_actor_context_required" + ) + + +def test_gateway_rag_search_uses_preexisting_actor_and_rejects_conflict( + monkeypatch, +): + from src.backend.integrations.internal_mcp import ( + InternalMCPGateway, + InternalMCPTransport, + build_default_catalogue, + ) + from src.backend.security.access_control import override_current_actor + + stub = _StubRAG( + results=[ + { + "id": "doc-1", + "text": "programme context", + "metadata": {}, + "score": 0.9, + } + ] + ) + monkeypatch.setattr( + "src.backend.services.rag_service.get_rag_service", + lambda *_args, **_kwargs: stub, + ) + gateway = InternalMCPGateway( + catalogue=build_default_catalogue(), + transport=InternalMCPTransport(), + enabled=True, + ) + + with override_current_actor( + user_concept_id="#V#authenticated_user", + organisation_concept_id="#V#authenticated_org", + ): + accepted = gateway.invoke( + "search_knowledge_base", + {"query": "programme context"}, + ).payload + denied = gateway.invoke( + "search_knowledge_base", + { + "query": "programme context", + "namespace": "#V#other_user@other_org", + "user_concept_id": "#V#other_user", + "organisation_concept_id": "#V#other_org", + }, + ).payload + + assert accepted["success"] is True + assert accepted["namespace"] == ( + "#V#authenticated_user@authenticated_org" + ) + assert accepted["namespace_source"] == "trusted_actor_context" + assert stub.last_permissions_context == { + "user_id": "#V#authenticated_user", + "organisation_concept_id": "#V#authenticated_org", + } + assert denied["success"] is False + assert denied["error"] == "namespace_mismatch" + assert set(denied["mismatch_fields"]) == { + "namespace", + "user_concept_id", + "organisation_concept_id", + } + + def test_search_knowledge_base_degrades_when_embedding_backend_fails(monkeypatch): from src.backend.integrations.internal_mcp import catalogue as cat diff --git a/tests/backend/test_rag_retrieval_state.py b/tests/backend/test_rag_retrieval_state.py index a414e3b91..ce34f0417 100644 --- a/tests/backend/test_rag_retrieval_state.py +++ b/tests/backend/test_rag_retrieval_state.py @@ -84,7 +84,7 @@ def test_llamaindex_query_preserves_blocked_namespace_state( assert results.retrieval_state["cause"] == "namespace_lock_contended" -def test_llamaindex_query_marks_only_completed_empty_search_as_valid( +def test_llamaindex_bounded_empty_search_is_inconclusive( monkeypatch: pytest.MonkeyPatch, ) -> None: class _Retriever: @@ -112,9 +112,16 @@ def as_retriever(self, *, similarity_top_k: int) -> _Retriever: results = rag.query("synthetic query", namespace="#V#test_user") assert results == [] - assert results.retrieval_state["status"] == "valid_empty" - assert results.retrieval_state["usable"] is True - assert results.retrieval_state["authoritative_empty"] is True + assert results.retrieval_state["status"] == "degraded" + assert results.retrieval_state["usable"] is False + assert results.retrieval_state["authoritative_empty"] is False + assert results.retrieval_state["cause"] == ( + "bounded_visible_result_window_inconclusive" + ) + assert results.retrieval_state["candidate_count"] == 0 + assert results.retrieval_state["candidate_limit"] == 5 + assert results.retrieval_state["candidate_limit_reached"] is False + assert "filtered_candidate_count" not in results.retrieval_state def test_filtered_candidate_window_is_not_reported_as_authoritative_empty( @@ -162,16 +169,20 @@ def as_retriever(self, *, similarity_top_k: int) -> _Retriever: ) assert results == [] - assert results.retrieval_state["status"] == "candidate_window_exhausted" + assert results.retrieval_state["status"] == "degraded" assert results.retrieval_state["authoritative_empty"] is False - assert results.retrieval_state["candidate_count"] == 50 - assert results.retrieval_state["filtered_candidate_count"] == 50 - assert results.retrieval_state["candidate_limit_reached"] is True + assert results.retrieval_state["cause"] == ( + "bounded_visible_result_window_inconclusive" + ) + assert results.retrieval_state["candidate_count"] == 0 + assert results.retrieval_state["candidate_limit"] == 5 + assert results.retrieval_state["candidate_limit_reached"] is False + assert "filtered_candidate_count" not in results.retrieval_state assert { item["action_type"] for item in results.retrieval_state["recovery_affordances"] } == { "retry", - "expand_candidate_window", + "inspect_runtime", } @@ -227,13 +238,200 @@ def as_retriever(self, *, similarity_top_k: int) -> _Retriever: assert results.retrieval_state["status"] == "partial_results" assert results.retrieval_state["usable"] is True assert results.retrieval_state["authoritative_empty"] is False - assert results.retrieval_state["filtered_candidate_count"] == 49 - assert results.retrieval_state["candidate_limit_reached"] is True + assert results.retrieval_state["cause"] == ( + "bounded_visible_result_window_underfilled" + ) + assert results.retrieval_state["candidate_count"] == 1 + assert results.retrieval_state["candidate_limit"] == 5 + assert results.retrieval_state["candidate_limit_reached"] is False + assert "filtered_candidate_count" not in results.retrieval_state assert { item["action_type"] for item in results.retrieval_state["recovery_affordances"] } == {"retry", "expand_candidate_window"} +def test_hidden_scoped_candidates_are_absent_from_retrieval_diagnostics( + monkeypatch: pytest.MonkeyPatch, +) -> None: + nodes = [ + SimpleNamespace( + node=SimpleNamespace( + metadata={ + "type": "scoped_knowledge_assertion", + "assertion_id": "ska_revoked", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ref_doc_id="scoped_assertion:ska_revoked", + node_id="node-scoped", + get_content=lambda: "revoked private assertion", + ), + score=0.9, + ), + SimpleNamespace( + node=SimpleNamespace( + metadata={ + "type": "text_relation", + "relation_id": "visible", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ref_doc_id="text_relation:visible", + node_id="node-visible", + get_content=lambda: "visible candidate", + ), + score=0.8, + ), + SimpleNamespace( + node=SimpleNamespace( + metadata={ + "type": "text_relation", + "relation_id": "filtered", + "user_id": "#V#other_user", + "organisation_concept_id": "#V#org", + }, + ref_doc_id="text_relation:filtered", + node_id="node-filtered", + get_content=lambda: "ordinary filtered candidate", + ), + score=0.7, + ), + ] + + class _Retriever: + def retrieve(self, _query_text: str) -> list[Any]: + return nodes + + class _Index: + def as_retriever(self, *, similarity_top_k: int) -> _Retriever: + assert similarity_top_k == 50 + return _Retriever() + + rag = _bare_rag_service(monkeypatch) + monkeypatch.setattr( + rag, + "get_namespace_runtime_state", + lambda _namespace: {"status": "compatible", "compatible": True}, + ) + monkeypatch.setattr(rag, "_maybe_load_index", lambda _namespace: _Index()) + monkeypatch.setattr( + rag, + "_temporarily_bound_query_embed_model", + lambda: (None, {}), + ) + monkeypatch.setattr( + "src.backend.services.scoped_rag_authority_service." + "current_authorised_rag_candidate_keys", + lambda *_args, **_kwargs: { + ("text_relation", "visible"), + ("text_relation", "filtered"), + }, + ) + + results = rag.query( + "synthetic query", + namespace="#V#target_user@org", + permissions_context={ + "mode": "concepts", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ) + + assert [row["id"] for row in results] == ["text_relation:visible"] + assert results.retrieval_state["status"] == "partial_results" + assert results.retrieval_state["cause"] == ( + "bounded_visible_result_window_underfilled" + ) + assert results.retrieval_state["candidate_count"] == 1 + assert results.retrieval_state["candidate_limit"] == 5 + assert results.retrieval_state["candidate_limit_reached"] is False + assert "filtered_candidate_count" not in results.retrieval_state + assert rag._last_query_info["retrieved"] == 1 + assert rag._last_query_info["returned"] == 1 + + +def test_hidden_authority_rows_match_clean_empty_retrieval_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + nodes = [ + SimpleNamespace( + node=SimpleNamespace( + metadata={ + "type": "scoped_knowledge_assertion", + "assertion_id": f"ska_hidden_{index}", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ref_doc_id=f"scoped_assertion:ska_hidden_{index}", + node_id=f"node-{index}", + get_content=lambda: "hidden", + ), + score=1.0, + ) + for index in range(50) + ] + + class _Retriever: + def retrieve(self, query_text: str) -> list[Any]: + return nodes if query_text == "hidden query" else [] + + class _Index: + def as_retriever(self, *, similarity_top_k: int) -> _Retriever: + assert similarity_top_k == 50 + return _Retriever() + + rag = _bare_rag_service(monkeypatch) + monkeypatch.setattr( + rag, + "get_namespace_runtime_state", + lambda _namespace: {"status": "compatible", "compatible": True}, + ) + monkeypatch.setattr(rag, "_maybe_load_index", lambda _namespace: _Index()) + monkeypatch.setattr( + rag, + "_temporarily_bound_query_embed_model", + lambda: (None, {}), + ) + monkeypatch.setattr( + "src.backend.services.scoped_rag_authority_service." + "current_authorised_rag_candidate_keys", + lambda *_args, **_kwargs: set(), + ) + + hidden_results = rag.query( + "hidden query", + namespace="#V#target_user@org", + permissions_context={ + "mode": "concepts", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ) + empty_results = rag.query( + "clean empty query", + namespace="#V#target_user@org", + permissions_context={ + "mode": "concepts", + "user_id": "#V#target_user", + "organisation_concept_id": "#V#org", + }, + ) + + assert hidden_results == empty_results == [] + assert hidden_results.retrieval_state == empty_results.retrieval_state + assert hidden_results.retrieval_state["status"] == "degraded" + assert hidden_results.retrieval_state["authoritative_empty"] is False + assert hidden_results.retrieval_state["cause"] == ( + "bounded_visible_result_window_inconclusive" + ) + assert hidden_results.retrieval_state["candidate_count"] == 0 + assert hidden_results.retrieval_state["candidate_limit"] == 5 + assert hidden_results.retrieval_state["candidate_limit_reached"] is False + assert "filtered_candidate_count" not in hidden_results.retrieval_state + assert rag._last_query_info["retrieved"] == 0 + + def test_search_knowledge_base_does_not_report_index_mismatch_as_success( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/backend/test_rag_service.py b/tests/backend/test_rag_service.py index f1a2c61dd..483fab574 100644 --- a/tests/backend/test_rag_service.py +++ b/tests/backend/test_rag_service.py @@ -3,10 +3,60 @@ import shutil import pytest +from bson import ObjectId from unittest.mock import MagicMock, patch from src.backend.services.rag_service import get_rag_service +def test_base_rag_candidate_rechecks_subject_and_predicate_visibility( + monkeypatch, +): + from src.backend.services import scoped_rag_authority_service as authority + + relation_id = ObjectId() + monkeypatch.setattr( + authority.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [ + { + "_id": relation_id, + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + } + ], + ) + visible = {"#V#subject", "#V#hasDescription"} + monkeypatch.setattr( + authority, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids) & visible, + ) + metadata = [ + { + "type": "text_relation", + "relation_id": str(relation_id), + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + } + ] + + assert authority.current_authorised_rag_candidate_keys( + metadata, + user_concept_id="#V#user", + organisation_concept_id="#V#org", + ) == {("text_relation", str(relation_id))} + + visible.remove("#V#hasDescription") + assert ( + authority.current_authorised_rag_candidate_keys( + metadata, + user_concept_id="#V#user", + organisation_concept_id="#V#org", + ) + == set() + ) + + # Mock LlamaIndex components to avoid real API calls and dependencies during unit tests @pytest.fixture def mock_llamaindex(): @@ -163,6 +213,201 @@ def test_query_honours_workflow_capability_retrieval_candidate_limit( mock_llamaindex["index"].as_retriever.assert_called_with(similarity_top_k=7) +def test_concepts_mode_includes_current_scoped_assertion_candidates( + mock_llamaindex, + monkeypatch, + workspace_tmp_path, +): + _configure_rag_runtime_settings(monkeypatch) + from src.backend.services.rag_backends.llamaindex_backend import ( + LlamaIndexRAGService, + ) + from src.backend.services import scoped_rag_authority_service + + rag = LlamaIndexRAGService( + persistence_dir=str(workspace_tmp_path / "rag_storage") + ) + rag.upsert_documents( + [ + { + "id": "scoped_assertion:ska_1", + "text": "Scoped programme context", + "metadata": {"type": "scoped_knowledge_assertion"}, + } + ], + namespace="#V#user@org", + ) + node = mock_llamaindex["index"].as_retriever.return_value.retrieve.return_value[0] + node.node.ref_doc_id = "scoped_assertion:ska_1" + node.node.metadata = { + "type": "scoped_knowledge_assertion", + "assertion_id": "ska_1", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "user_id": "#V#user", + "organisation_concept_id": "#V#org", + } + authority_calls = [] + monkeypatch.setattr( + scoped_rag_authority_service, + "current_authorised_rag_candidate_keys", + lambda rows, **kwargs: authority_calls.append((rows, kwargs)) + or {("scoped_knowledge_assertion", "ska_1")}, + ) + + results = rag.query( + "programme context", + namespace="#V#user@org", + permissions_context={ + "mode": "concepts", + "user_id": "#V#user", + "organisation_concept_id": "#V#org", + }, + ) + + assert [row["id"] for row in results] == ["scoped_assertion:ska_1"] + assert authority_calls == [ + ( + [node.node.metadata], + { + "user_concept_id": "#V#user", + "organisation_concept_id": "#V#org", + }, + ) + ] + + +def test_scoped_rag_hit_is_live_filtered_after_assertion_revocation( + mock_llamaindex, + monkeypatch, + workspace_tmp_path, +): + import mongomock + + _configure_rag_runtime_settings(monkeypatch) + from src.backend.services.rag_backends.llamaindex_backend import ( + LlamaIndexRAGService, + ) + from src.backend.services import scoped_rag_authority_service + + collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + collection.insert_one( + { + "assertion_id": "ska_live", + "status": "asserted", + "object_kind": "text", + "scope": {"audience_keys": ["org:#V#org"]}, + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + } + ) + monkeypatch.setattr( + scoped_rag_authority_service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + visible_concepts = {"#V#subject", "#V#hasDescription"} + monkeypatch.setattr( + scoped_rag_authority_service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids) & visible_concepts, + ) + + rag = LlamaIndexRAGService( + persistence_dir=str(workspace_tmp_path / "rag_storage") + ) + rag.upsert_documents( + [ + { + "id": "scoped_assertion:ska_live", + "text": "Scoped programme context", + "metadata": {"type": "scoped_knowledge_assertion"}, + } + ], + namespace="#V#user@org", + ) + node = mock_llamaindex["index"].as_retriever.return_value.retrieve.return_value[0] + node.node.ref_doc_id = "scoped_assertion:ska_live" + node.node.metadata = { + "type": "scoped_knowledge_assertion", + "assertion_id": "ska_live", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "user_id": "#V#user", + "organisation_concept_id": "#V#org", + } + permissions = { + "mode": "concepts", + "user_id": "#V#user", + "organisation_concept_id": "#V#org", + } + + assert len( + rag.query( + "programme context", + namespace="#V#user@org", + permissions_context=permissions, + ) + ) == 1 + + collection.update_one( + {"assertion_id": "ska_live"}, + {"$set": {"status": "retracted"}}, + ) + assert ( + rag.query( + "programme context", + namespace="#V#user@org", + permissions_context=permissions, + ) + == [] + ) + + collection.update_one( + {"assertion_id": "ska_live"}, + {"$set": {"status": "asserted"}}, + ) + visible_concepts.remove("#V#hasDescription") + assert ( + rag.query( + "programme context", + namespace="#V#user@org", + permissions_context=permissions, + ) + == [] + ) + + visible_concepts.add("#V#hasDescription") + collection.update_one( + {"assertion_id": "ska_live"}, + {"$set": {"scope.audience_keys": ["org:#V#other"]}}, + ) + assert ( + rag.query( + "programme context", + namespace="#V#user@org", + permissions_context=permissions, + ) + == [] + ) + + collection.update_one( + {"assertion_id": "ska_live"}, + {"$set": {"scope.audience_keys": ["org:#V#org"]}}, + ) + visible_concepts.remove("#V#subject") + assert ( + rag.query( + "programme context", + namespace="#V#user@org", + permissions_context=permissions, + ) + == [] + ) + + def test_delete_documents(mock_llamaindex): service = get_rag_service("llamaindex") count = service.delete_documents(["doc1"]) diff --git a/tests/backend/test_rag_text_relation_change_hook_service.py b/tests/backend/test_rag_text_relation_change_hook_service.py new file mode 100644 index 000000000..7fc77162a --- /dev/null +++ b/tests/backend/test_rag_text_relation_change_hook_service.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import queue +import threading + +import pytest + +_QUEUE_FIELDS = { + "mechanism": "in_memory_bounded_queue", + "durable": False, +} + + +class _DeferredThread: + def __init__(self, *, target, name, daemon): + self.target = target + self.name = name + self.daemon = daemon + self.started = False + + def start(self): + self.started = True + + def is_alive(self): + return self.started + + +def _install_deferred_worker_state(monkeypatch, service, *, queue_size=4): + work_queue = queue.Queue(maxsize=queue_size) + pending_keys = set() + threads = [] + + def _thread_factory(**kwargs): + thread = _DeferredThread(**kwargs) + threads.append(thread) + return thread + + monkeypatch.setattr(service, "_RAG_SYNC_QUEUE", work_queue) + monkeypatch.setattr(service, "_RAG_SYNC_PENDING_KEYS", pending_keys) + monkeypatch.setattr(service, "_RAG_SYNC_RERUN_KEYS", set()) + monkeypatch.setattr(service, "_RAG_SYNC_LOCK", threading.Lock()) + monkeypatch.setattr(service, "_RAG_SYNC_WORKER_THREAD", None) + monkeypatch.setattr(service.threading, "Thread", _thread_factory) + return work_queue, pending_keys, threads + + +def test_post_write_hook_schedules_without_running_rag_inline(monkeypatch): + from src.backend.services import rag_text_relation_change_hook_service as service + + work_queue, pending_keys, threads = _install_deferred_worker_state( + monkeypatch, + service, + ) + + def _must_not_run_inline(**_kwargs): + raise AssertionError("derived RAG maintenance ran on the primary write path") + + monkeypatch.setattr( + service, + "_run_concept_text_relation_rag_sync", + _must_not_run_inline, + ) + + result = service.maybe_sync_concept_text_relations_to_rag( + namespace=" #V#user@org ", + concept_id=" #V#subject ", + predicate=" hasDescription ", + ) + + assert result == {**_QUEUE_FIELDS, "success": True, "scheduled": True} + assert len(threads) == 1 + assert threads[0].started is True + assert threads[0].daemon is True + assert work_queue.get_nowait() == { + "namespace": "#V#user@org", + "concept_id": "#V#subject", + "predicate": "hasDescription", + "scoped_assertion_id": None, + } + assert pending_keys == { + ("#V#user@org", "#V#subject", "hasDescription", None), + } + + +def test_post_write_hook_coalesces_duplicate_pending_refresh(monkeypatch): + from src.backend.services import rag_text_relation_change_hook_service as service + + work_queue, _pending_keys, threads = _install_deferred_worker_state( + monkeypatch, + service, + ) + + first = service.maybe_sync_concept_text_relations_to_rag( + namespace="#V#user@org", + concept_id="#V#subject", + predicate="hasDescription", + ) + second = service.maybe_sync_concept_text_relations_to_rag( + namespace="#V#user@org", + concept_id="#V#subject", + predicate="hasDescription", + ) + + assert first == {**_QUEUE_FIELDS, "success": True, "scheduled": True} + assert second == { + **_QUEUE_FIELDS, + "success": True, + "scheduled": False, + "reason": "already_scheduled", + "rerun_requested": True, + } + assert len(threads) == 1 + assert work_queue.qsize() == 1 + assert service._RAG_SYNC_RERUN_KEYS == { + ("#V#user@org", "#V#subject", "hasDescription", None), + } + + +def test_post_write_hook_reports_queue_saturation_without_raising(monkeypatch): + from src.backend.services import rag_text_relation_change_hook_service as service + + work_queue, pending_keys, _threads = _install_deferred_worker_state( + monkeypatch, + service, + queue_size=1, + ) + work_queue.put_nowait( + { + "namespace": "#V#other@org", + "concept_id": "#V#other", + "predicate": None, + "scoped_assertion_id": None, + } + ) + + result = service.maybe_sync_concept_text_relations_to_rag( + namespace="#V#user@org", + concept_id="#V#subject", + ) + + assert result == { + **_QUEUE_FIELDS, + "success": False, + "scheduled": False, + "skipped": False, + "reason": "queue_full", + } + assert ("#V#user@org", "#V#subject", None, None) not in pending_keys + + +def test_background_worker_executes_job_and_releases_pending_key(monkeypatch): + from src.backend.services import rag_text_relation_change_hook_service as service + + job = { + "namespace": "#V#user@org", + "concept_id": "#V#subject", + "predicate": "hasDescription", + "scoped_assertion_id": None, + } + + class _OneJobQueue: + def __init__(self): + self.returned = False + self.task_done_calls = 0 + + def get(self): + if self.returned: + raise StopIteration + self.returned = True + return job + + def task_done(self): + self.task_done_calls += 1 + + work_queue = _OneJobQueue() + pending_key = ("#V#user@org", "#V#subject", "hasDescription", None) + calls = [] + monkeypatch.setattr(service, "_RAG_SYNC_QUEUE", work_queue) + monkeypatch.setattr(service, "_RAG_SYNC_PENDING_KEYS", {pending_key}) + monkeypatch.setattr(service, "_RAG_SYNC_RERUN_KEYS", set()) + monkeypatch.setattr(service, "_RAG_SYNC_LOCK", threading.Lock()) + monkeypatch.setattr( + service, + "_run_concept_text_relation_rag_sync", + lambda **kwargs: calls.append(kwargs) or {"success": True}, + ) + + with pytest.raises(StopIteration): + service._concept_text_relation_rag_sync_worker() + + assert calls == [ + { + "namespace": "#V#user@org", + "concept_id": "#V#subject", + "predicate": "hasDescription", + "scoped_assertion_id": None, + } + ] + assert service._RAG_SYNC_PENDING_KEYS == set() + assert work_queue.task_done_calls == 1 + + +def test_background_worker_requeues_once_when_write_arrives_during_refresh( + monkeypatch, +): + from src.backend.services import rag_text_relation_change_hook_service as service + + job = { + "namespace": "#V#user@org", + "concept_id": "#V#subject", + "predicate": "hasDescription", + "scoped_assertion_id": None, + } + + class _OneJobQueue: + def __init__(self): + self.returned = False + self.requeued = [] + + def get(self): + if self.returned: + raise StopIteration + self.returned = True + return job + + def put_nowait(self, value): + self.requeued.append(value) + + def task_done(self): + return None + + work_queue = _OneJobQueue() + pending_key = ("#V#user@org", "#V#subject", "hasDescription", None) + monkeypatch.setattr(service, "_RAG_SYNC_QUEUE", work_queue) + monkeypatch.setattr(service, "_RAG_SYNC_PENDING_KEYS", {pending_key}) + monkeypatch.setattr(service, "_RAG_SYNC_RERUN_KEYS", {pending_key}) + monkeypatch.setattr(service, "_RAG_SYNC_LOCK", threading.Lock()) + monkeypatch.setattr( + service, + "_run_concept_text_relation_rag_sync", + lambda **_kwargs: {"success": True}, + ) + + with pytest.raises(StopIteration): + service._concept_text_relation_rag_sync_worker() + + assert work_queue.requeued == [job] + assert service._RAG_SYNC_PENDING_KEYS == {pending_key} + assert service._RAG_SYNC_RERUN_KEYS == set() + + +def test_exact_scoped_refresh_does_not_spend_budget_on_base_rows(monkeypatch): + from src.backend.services import rag_text_relation_change_hook_service as service + from src.backend.services import rag_text_relation_sync_service as sync_service + + calls = [] + monkeypatch.setattr( + sync_service, + "sync_scoped_assertions_to_rag", + lambda **kwargs: calls.append(kwargs) or {"success": True}, + ) + monkeypatch.setattr( + sync_service, + "sync_text_relations_to_rag", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("exact scoped refresh must not scan base rows") + ), + ) + + result = service._run_concept_text_relation_rag_sync( + namespace="#V#user@org", + concept_id="#V#subject", + predicate="hasDescription", + scoped_assertion_id="ska_exact", + ) + + assert result == {"success": True} + assert calls == [ + { + "namespace": "#V#user@org", + "assertion_ids": ["ska_exact"], + } + ] diff --git a/tests/backend/test_rag_text_relation_sync_service.py b/tests/backend/test_rag_text_relation_sync_service.py index 2c3388390..38fd50d1f 100644 --- a/tests/backend/test_rag_text_relation_sync_service.py +++ b/tests/backend/test_rag_text_relation_sync_service.py @@ -1,7 +1,11 @@ from __future__ import annotations +from datetime import datetime, timezone from typing import Any, Dict, Optional +import mongomock +from bson import ObjectId + class _ConceptsCollection: def __init__(self, visible_concepts: set[str]): @@ -37,18 +41,295 @@ def _extract_concept_id(q: Any) -> Optional[str]: class _StubRAG: def __init__(self): self.upserts = [] + self.deletes = [] + self.delete_batches = [] def upsert_documents(self, docs, *, namespace=None, allow_partial_failures=True): self.upserts.extend(docs) return (len(docs), 0) + def delete_documents(self, ids, *, namespace=None): + batch = list(ids) + self.delete_batches.append(batch) + self.deletes.extend(batch) + return len(batch) + + +def test_one_pass_iterator_opens_each_store_once_and_batches_visibility( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + text_value_ids = [ObjectId(), ObjectId()] + base_relations = [ + { + "_id": ObjectId(), + "subject_concept_id": f"#V#subject_{index}", + "predicate": "hasDescription", + "object_text_id": text_value_ids[index], + } + for index in range(2) + ] + base_find_calls = [] + + def _find_base(query, **kwargs): + base_find_calls.append((query, kwargs)) + return iter(base_relations) + + monkeypatch.setattr(svc.TextRelationsRepository, "find", _find_base) + text_value_find_calls = [] + + def _find_text_values(query, _projection=None, **_kwargs): + text_value_find_calls.append(query) + return [ + { + "_id": text_value_id, + "text": f"Text {text_value_id}", + "lang": "en-NZ", + } + for text_value_id in text_value_ids + ] + + monkeypatch.setattr( + svc.TextValuesRepository, + "find", + _find_text_values, + ) + monkeypatch.setattr( + svc.TextValuesRepository, + "find_one", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("iterator must batch TextValue hydration") + ), + ) + + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + for index in range(2): + scoped_collection.insert_one( + { + "assertion_id": f"ska_{index}", + "subject_concept_id": f"#V#subject_{index}", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": f"Scoped {index}", + "language": "en-NZ", + }, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + } + ) + + class _CountingCollection: + def __init__(self, delegate): + self.delegate = delegate + self.find_calls = [] + + def find(self, query): + self.find_calls.append(query) + return self.delegate.find(query) + + counting_scoped = _CountingCollection(scoped_collection) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: counting_scoped, + ) + visibility_batches = [] + + def _visible(concept_ids, **_kwargs): + batch = set(concept_ids) + visibility_batches.append(batch) + return batch + + monkeypatch.setattr(svc, "_visible_concept_ids_in_namespace", _visible) + + docs = list( + svc.iter_text_relation_docs_for_namespace( + namespace="#V#user@org", + source_batch_size=2, + ) + ) + + assert len(base_find_calls) == 1 + assert base_find_calls[0][1]["sort"] == [("_id", 1)] + assert len(text_value_find_calls) == 1 + assert set(text_value_find_calls[0]["_id"]["$in"]) == set(text_value_ids) + assert len(counting_scoped.find_calls) == 1 + assert [doc.doc_id.split(":", 1)[0] for doc in docs] == [ + "text_relation", + "text_relation", + "scoped_assertion", + "scoped_assertion", + ] + assert visibility_batches == [ + { + "#V#subject_0", + "#V#subject_1", + svc.predicate_concept_id_for_storage("hasDescription"), + }, + { + "#V#subject_0", + "#V#subject_1", + svc.predicate_concept_id_for_storage("hasDescription"), + }, + ] + + +def test_iterator_releases_working_caches_between_raw_batches(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + text_value_ids = [ObjectId() for _ in range(4)] + base_relations = [ + { + "_id": ObjectId(), + "subject_concept_id": f"#V#subject_{index}", + "predicate": "hasDescription", + "object_text_id": text_value_ids[index], + } + for index in range(4) + ] + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: iter(base_relations), + ) + monkeypatch.setattr( + svc.TextValuesRepository, + "find", + lambda query, *_args, **_kwargs: [ + { + "_id": text_value_id, + "text": f"Text {text_value_id}", + "lang": "en-NZ", + } + for text_value_id in query["_id"]["$in"] + ], + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: None, + ) + visibility_batches = [] + + def _visible(concept_ids, **_kwargs): + batch = set(concept_ids) + visibility_batches.append(batch) + return batch + + monkeypatch.setattr(svc, "_visible_concept_ids_in_namespace", _visible) + cache_sizes = [] + original_projection = svc._base_relation_to_rag_doc + + def _capture_cache_sizes(relation, **kwargs): + cache_sizes.append( + ( + len(kwargs["concept_visibility"]), + len(kwargs["text_value_cache"]), + ) + ) + return original_projection(relation, **kwargs) + + monkeypatch.setattr( + svc, + "_base_relation_to_rag_doc", + _capture_cache_sizes, + ) + + docs = list( + svc.iter_text_relation_docs_for_namespace( + namespace="#V#user@org", + source_batch_size=2, + ) + ) + + predicate_id = svc.predicate_concept_id_for_storage("hasDescription") + assert len(docs) == 4 + assert cache_sizes == [(3, 2), (3, 2), (3, 2), (3, 2)] + assert visibility_batches == [ + {"#V#subject_0", "#V#subject_1", predicate_id}, + {"#V#subject_2", "#V#subject_3", predicate_id}, + ] + + +def test_iterator_flushes_stale_only_raw_batches(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + text_value_ids = [ObjectId() for _ in range(5)] + base_relations = [ + { + "_id": f"r{index}", + "subject_concept_id": f"#V#blocked_{index}", + "predicate": "hasDescription", + "object_text_id": text_value_ids[index], + } + for index in range(5) + ] + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: iter(base_relations), + ) + monkeypatch.setattr( + svc.TextValuesRepository, + "find", + lambda query, *_args, **_kwargs: [ + { + "_id": text_value_id, + "text": f"Text {text_value_id}", + "lang": "en-NZ", + } + for text_value_id in query["_id"]["$in"] + ], + ) + predicate_id = svc.predicate_concept_id_for_storage("hasDescription") + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids) & {predicate_id}, + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: None, + ) + flushed = [] + + docs = list( + svc.iter_text_relation_docs_for_namespace( + namespace="#V#user@org", + source_batch_size=2, + stale_doc_sink=lambda stale_ids: flushed.append(list(stale_ids)), + ) + ) + + assert docs == [] + assert flushed == [ + ["text_relation:r0", "text_relation:r1"], + ["text_relation:r2", "text_relation:r3"], + ["text_relation:r4"], + ] + def test_sync_text_relations_indexes_visible_concepts(monkeypatch): from src.backend.services import rag_text_relation_sync_service as svc # Visible concept set for the namespace. monkeypatch.setattr( - svc, "get_concepts_collection", lambda: _ConceptsCollection({"#V#allowed"}) + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: { + concept_id + for concept_id in concept_ids + if concept_id + in { + "#V#allowed", + svc.predicate_concept_id_for_storage("hasDescription"), + } + }, ) # Two relations: one visible, one not. @@ -70,17 +351,28 @@ def test_sync_text_relations_indexes_visible_concepts(monkeypatch): ][skip : (skip + limit) if limit else None], ) - def _tv_find_one(filter_doc, _projection=None): - oid = filter_doc.get("_id") - if str(oid) == "000000000000000000000001": - return {"_id": oid, "text": "Hello world", "lang": "en-NZ"} - if str(oid) == "000000000000000000000002": - return {"_id": oid, "text": "Should not index", "lang": "en-NZ"} - return None + def _tv_find(filter_doc, _projection=None, **_kwargs): + return [ + { + "_id": oid, + "text": ( + "Hello world" + if str(oid).endswith("1") + else "Should not index" + ), + "lang": "en-NZ", + } + for oid in filter_doc["_id"]["$in"] + ] monkeypatch.setattr( - "src.backend.db.repositories.text_value_repository.TextValuesRepository.find_one", - _tv_find_one, + "src.backend.db.repositories.text_value_repository.TextValuesRepository.find", + _tv_find, + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: None, ) rag = _StubRAG() @@ -103,6 +395,8 @@ def _tv_find_one(filter_doc, _projection=None): assert meta["predicate"] == "hasDescription" assert meta["relation_id"] == "r1" assert meta["language"] == "en-NZ" + assert rag.deletes == ["text_relation:r2"] + assert result["deleted"] == 1 # Required for permission filtering. assert meta["user_id"] == "#V#user" @@ -110,44 +404,744 @@ def _tv_find_one(filter_doc, _projection=None): assert meta["org_id"] == "#V#org" +def test_full_sync_deletes_stale_only_batch_in_bounded_chunks(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + text_value_ids = [ObjectId() for _ in range(5)] + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: iter( + [ + { + "_id": f"r{index}", + "subject_concept_id": f"#V#blocked_{index}", + "predicate": "hasDescription", + "object_text_id": text_value_ids[index], + } + for index in range(5) + ] + ), + ) + monkeypatch.setattr( + svc.TextValuesRepository, + "find", + lambda query, *_args, **_kwargs: [ + { + "_id": text_value_id, + "text": f"Text {text_value_id}", + "lang": "en-NZ", + } + for text_value_id in query["_id"]["$in"] + ], + ) + predicate_id = svc.predicate_concept_id_for_storage("hasDescription") + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids) & {predicate_id}, + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: None, + ) + rag = _StubRAG() + monkeypatch.setattr(svc, "get_rag_service", lambda *_args, **_kwargs: rag) + + result = svc.sync_text_relations_to_rag( + namespace="#V#user@org", + limit=100, + batch_size=2, + ) + + assert result == { + "success": True, + "namespace": "#V#user@org", + "total_candidates": 0, + "added": 0, + "failed": 0, + "deleted": 5, + } + assert rag.upserts == [] + assert rag.delete_batches == [ + ["text_relation:r0", "text_relation:r1"], + ["text_relation:r2", "text_relation:r3"], + ["text_relation:r4"], + ] + + def test_collect_includes_scoped_assertion_for_namespace(monkeypatch): from src.backend.services import rag_text_relation_sync_service as svc - from src.backend.services import scoped_assertion_service monkeypatch.setattr( svc.TextRelationsRepository, "find", lambda *_args, **_kwargs: [], ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_123", + "subject_concept_id": "#V#gillian_dobbie", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": "Organisation-scoped programme context.", + "language": "en-NZ", + }, + "scope": { + "mode": "organisation", + "audience_keys": ["org:#V#org"], + }, + "provenance": { + "asserted_by_user_concept_id": "#V#asserting_user", + "organisation_concept_id": "#V#org", + "turn_id": "turn-123", + }, + "status": "asserted", + "canonical_publication": False, + "updated_at": datetime(2026, 7, 29, tzinfo=timezone.utc), + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids), + ) + + docs = svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + ) + + assert len(docs) == 1 + assert docs[0].doc_id == "scoped_assertion:ska_123" + assert "Organisation-scoped programme context." in docs[0].text + assert docs[0].metadata["type"] == "scoped_knowledge_assertion" + assert docs[0].metadata["assertion_id"] == "ska_123" + assert docs[0].metadata["relation_id"] is None + assert docs[0].metadata["row_kind"] == "scoped_assertion" + assert docs[0].metadata["canonical_publication"] is False + assert docs[0].metadata["organisation_concept_id"] == "#V#org" + assert docs[0].metadata["audience_user_concept_id"] == "#V#user" + assert docs[0].metadata["audience_organisation_concept_id"] == "#V#org" + assert docs[0].metadata["audience_keys"] == ["org:#V#org"] + assert docs[0].metadata["asserted_by_user_concept_id"] == "#V#asserting_user" + assert docs[0].metadata["assertion_provenance"]["turn_id"] == "turn-123" + + +def test_combined_pages_respect_one_budget_without_repeating_scoped_rows( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + base_relations = [ + { + "_id": f"r{index}", + "subject_concept_id": "#V#allowed", + "predicate": "hasDescription", + "object_text_id": f"{index + 1:024d}", + } + for index in range(2) + ] + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda _filter, projection=None, sort=None, skip=0, limit=0: base_relations[ + skip : skip + limit + ], + ) monkeypatch.setattr( - scoped_assertion_service, - "list_visible_scoped_assertions", - lambda **_kwargs: [ + svc.TextValuesRepository, + "find", + lambda query, _projection=None, **_kwargs: [ { - "assertion_id": "ska_123", - "subject_concept_id": "#V#gillian_dobbie", + "_id": object_id, + "text": f"Base {str(object_id)[-1]}", + "lang": "en-NZ", + } + for object_id in query["_id"]["$in"] + ], + ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids), + ) + + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + for index in range(3): + scoped_collection.insert_one( + { + "assertion_id": f"ska_{index}", + "subject_concept_id": "#V#allowed", "predicate": "hasDescription", "object_kind": "text", "object_text": { - "text": "Organisation-scoped programme context.", + "text": f"Scoped {index}", "language": "en-NZ", }, - "scope": { - "mode": "organisation", - "audience_keys": ["org:#V#org"], + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + "updated_at": datetime( + 2026, + 7, + 29, + 12 - index, + tzinfo=timezone.utc, + ), + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + + page_one = svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + skip=0, + limit=2, + ) + page_two = svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + skip=2, + limit=2, + ) + page_three = svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + skip=4, + limit=2, + ) + + assert [doc.doc_id for doc in page_one] == ["text_relation:r0", "text_relation:r1"] + assert [doc.doc_id for doc in page_two] == [ + "scoped_assertion:ska_0", + "scoped_assertion:ska_1", + ] + assert [doc.doc_id for doc in page_three] == ["scoped_assertion:ska_2"] + all_ids = [ + doc.doc_id for page in (page_one, page_two, page_three) for doc in page + ] + assert len(all_ids) == len(set(all_ids)) + assert all(len(page) <= 2 for page in (page_one, page_two, page_three)) + + +def test_scoped_visible_pages_scan_past_revoked_raw_rows_without_duplicates( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + subjects = ["#V#revoked", "#V#a", "#V#b", "#V#c"] + for index, subject_id in enumerate(subjects): + scoped_collection.insert_one( + { + "assertion_id": f"ska_{index}", + "subject_concept_id": subject_id, + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": f"Scoped {subject_id}", + "language": "en-NZ", }, - "canonical_publication": False, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + "updated_at": datetime( + 2026, + 7, + 29, + 12 - index, + tzinfo=timezone.utc, + ), } - ], + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: { + concept_id + for concept_id in concept_ids + if concept_id != "#V#revoked" + }, + ) + + pages = [ + svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + skip=offset, + limit=1, + ) + for offset in range(3) + ] + + assert [[doc.doc_id for doc in page] for page in pages] == [ + ["scoped_assertion:ska_1"], + ["scoped_assertion:ska_2"], + ["scoped_assertion:ska_3"], + ] + all_ids = [doc.doc_id for page in pages for doc in page] + assert len(all_ids) == len(set(all_ids)) + + +def test_scoped_projection_requires_current_subject_and_predicate_visibility( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_hidden_predicate", + "subject_concept_id": "#V#subject", + "predicate": "#V#private_predicate", + "object_kind": "text", + "object_text": {"text": "Scoped.", "language": "en-NZ"}, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + "updated_at": datetime(2026, 7, 29, tzinfo=timezone.utc), + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + checked = [] + + def _visible(concept_ids, **_kwargs): + checked.append(set(concept_ids)) + return { + concept_id + for concept_id in concept_ids + if concept_id != "#V#private_predicate" + } + + monkeypatch.setattr(svc, "_visible_concept_ids_in_namespace", _visible) docs = svc.collect_text_relation_docs_for_namespace( namespace="#V#user@org", + limit=1, ) - assert len(docs) == 1 - assert docs[0].doc_id == "scoped_assertion:ska_123" - assert "Organisation-scoped programme context." in docs[0].text - assert docs[0].metadata["type"] == "scoped_knowledge_assertion" - assert docs[0].metadata["canonical_publication"] is False - assert docs[0].metadata["organisation_concept_id"] == "#V#org" + assert docs == [] + assert checked == [{"#V#subject", "#V#private_predicate"}] + + +def test_collect_limit_zero_does_not_query_either_store(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + def _unexpected_query(*_args, **_kwargs): + raise AssertionError("limit=0 must not query a backing store") + + monkeypatch.setattr(svc.TextRelationsRepository, "find", _unexpected_query) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + _unexpected_query, + ) + + assert ( + svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + limit=0, + ) + == [] + ) + + +def test_scoped_query_uses_audience_status_time_shape_and_mapped_page(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + class _Cursor: + def __init__(self): + self.sort_spec = None + self.skip_value = None + self.limit_value = None + + def sort(self, spec): + self.sort_spec = spec + return self + + def skip(self, value): + self.skip_value = value + return self + + def limit(self, value): + self.limit_value = value + return self + + def __iter__(self): + return iter(()) + + class _Collection: + def __init__(self): + self.query = None + self.cursor = _Cursor() + + def find(self, query): + self.query = query + return self.cursor + + collection = _Collection() + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + updated_since = datetime(2026, 7, 1, tzinfo=timezone.utc) + + svc.collect_text_relation_docs_for_namespace( + namespace="#V#user@org", + predicates=["hasDescription"], + languages=["en-NZ"], + updated_since=updated_since, + skip=7, + limit=3, + ) + + assert collection.query == { + "scope.audience_keys": { + "$in": ["user:#V#user", "org:#V#org"], + }, + "status": "asserted", + "object_kind": "text", + "updated_at": {"$gte": updated_since}, + "predicate": {"$in": ["hasDescription"]}, + "object_text.language": {"$in": ["en-NZ"]}, + } + assert collection.cursor.sort_spec == [ + ("updated_at", -1), + ("assertion_id", 1), + ] + # Public offsets count visible projections. The raw cursor starts at zero + # and scans through revoked/inaccessible candidates until that visible + # offset and page budget are satisfied or storage is exhausted. + assert collection.cursor.skip_value == 0 + assert collection.cursor.limit_value == 10 + + +def test_sync_reindexes_scoped_rows_when_namespace_has_no_base_rows(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids), + ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_only", + "subject_concept_id": "#V#only_scoped", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Only scoped.", "language": "en-NZ"}, + "scope": {"audience_keys": ["user:#V#user"]}, + "status": "asserted", + "updated_at": datetime(2026, 7, 29, tzinfo=timezone.utc), + } + ) + scoped_collection.insert_one( + { + "assertion_id": "ska_retracted", + "subject_concept_id": "#V#only_scoped", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": { + "text": "Retracted scoped.", + "language": "en-NZ", + }, + "scope": {"audience_keys": ["user:#V#user"]}, + "status": "retracted", + "updated_at": datetime(2026, 7, 29, tzinfo=timezone.utc), + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + rag = _StubRAG() + monkeypatch.setattr(svc, "get_rag_service", lambda *_args, **_kwargs: rag) + + result = svc.sync_text_relations_to_rag( + namespace="#V#user@org", + limit=10, + ) + + assert result == { + "success": True, + "namespace": "#V#user@org", + "total_candidates": 1, + "added": 1, + "failed": 0, + "deleted": 1, + } + assert [doc["id"] for doc in rag.upserts] == ["scoped_assertion:ska_only"] + assert rag.deletes == ["scoped_assertion:ska_retracted"] + + +def test_list_index_items_includes_scoped_candidates_without_relation_id( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: [], + ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids), + ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_listed", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Listed scoped.", "language": "en-NZ"}, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + "updated_at": datetime(2026, 7, 29, tzinfo=timezone.utc), + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + + report = svc.list_text_relation_index_items( + namespace="#V#user@org", + limit=10, + scan_limit=10, + ) + + assert report["total"] == 1 + assert len(report["items"]) == 1 + item = report["items"][0] + assert item["index_item_id"] == "scoped_assertion:ska_listed" + assert item["row_kind"] == "scoped_assertion" + assert item["relation_id"] is None + assert item["assertion_id"] == "ska_listed" + assert item["preview_length"] == len("Listed scoped.") + assert item["source"] == "scoped_knowledge_assertion" + + +def test_get_preview_supports_scoped_identifier_with_live_authority_checks( + monkeypatch, +): + from src.backend.services import rag_text_relation_sync_service as svc + + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_preview", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Scoped preview.", "language": "en-NZ"}, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + visible_concepts = {"#V#subject", "#V#hasDescription"} + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids) & visible_concepts, + ) + + prefixed = svc.get_text_relation_preview( + namespace="#V#user@org", + relation_id="scoped_assertion:ska_preview", + ) + exact = svc.get_text_relation_preview( + namespace="#V#user@org", + relation_id="ska_preview", + ) + + assert prefixed is not None + assert exact is not None + assert prefixed.doc_id == "scoped_assertion:ska_preview" + assert prefixed.metadata["assertion_id"] == "ska_preview" + assert prefixed.metadata["relation_id"] is None + + scoped_collection.update_one( + {"assertion_id": "ska_preview"}, + {"$set": {"status": "retracted"}}, + ) + assert ( + svc.get_text_relation_preview( + namespace="#V#user@org", + relation_id="ska_preview", + ) + is None + ) + + scoped_collection.update_one( + {"assertion_id": "ska_preview"}, + { + "$set": { + "status": "asserted", + "scope.audience_keys": ["org:#V#other"], + } + }, + ) + assert ( + svc.get_text_relation_preview( + namespace="#V#user@org", + relation_id="ska_preview", + ) + is None + ) + + scoped_collection.update_one( + {"assertion_id": "ska_preview"}, + {"$set": {"scope.audience_keys": ["org:#V#org"]}}, + ) + visible_concepts.remove("#V#hasDescription") + assert ( + svc.get_text_relation_preview( + namespace="#V#user@org", + relation_id="ska_preview", + ) + is None + ) + + +def test_exact_scoped_sync_bypasses_base_first_scan_budget(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + monkeypatch.setattr( + svc.TextRelationsRepository, + "find", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("exact scoped sync must not scan base relations") + ), + ) + monkeypatch.setattr( + svc, + "_visible_concept_ids_in_namespace", + lambda concept_ids, **_kwargs: set(concept_ids), + ) + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_exact", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Exact scoped.", "language": "en-NZ"}, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "asserted", + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + rag = _StubRAG() + monkeypatch.setattr(svc, "get_rag_service", lambda *_args, **_kwargs: rag) + + result = svc.sync_scoped_assertions_to_rag( + namespace="#V#user@org", + assertion_ids=["ska_exact"], + ) + + assert result["success"] is True + assert result["sync_scope"] == "exact_scoped_assertions" + assert [row["id"] for row in rag.upserts] == ["scoped_assertion:ska_exact"] + + +def test_exact_scoped_sync_removes_retracted_derived_document(monkeypatch): + from src.backend.services import rag_text_relation_sync_service as svc + + scoped_collection = mongomock.MongoClient()["von_test"][ + "scoped_knowledge_assertions" + ] + scoped_collection.insert_one( + { + "assertion_id": "ska_retracted", + "subject_concept_id": "#V#subject", + "predicate": "hasDescription", + "object_kind": "text", + "object_text": {"text": "Retracted.", "language": "en-NZ"}, + "scope": {"audience_keys": ["org:#V#org"]}, + "status": "retracted", + } + ) + monkeypatch.setattr( + svc, + "get_scoped_knowledge_assertions_collection", + lambda: scoped_collection, + ) + rag = _StubRAG() + monkeypatch.setattr(svc, "get_rag_service", lambda *_args, **_kwargs: rag) + + result = svc.sync_scoped_assertions_to_rag( + namespace="#V#user@org", + assertion_ids=["ska_retracted"], + ) + + assert result["success"] is True + assert result["added"] == 0 + assert result["deleted"] == 1 + assert rag.deletes == ["scoped_assertion:ska_retracted"] diff --git a/tests/backend/test_rag_text_relations_mcp_read_tools.py b/tests/backend/test_rag_text_relations_mcp_read_tools.py index 34a243f59..2e8f7b6d4 100644 --- a/tests/backend/test_rag_text_relations_mcp_read_tools.py +++ b/tests/backend/test_rag_text_relations_mcp_read_tools.py @@ -14,17 +14,32 @@ def test_rag_list_indexed_supports_vontology_text_relations(monkeypatch): lambda **_kwargs: { "items": [ { + "index_item_id": "text_relation:r1", + "row_kind": "base_text_relation", "relation_id": "r1", + "assertion_id": None, "subject_concept_id": "#V#x", "predicate": "hasDescription", "text_value_id": "t1", "lang": "en-NZ", "preview_length": 12, "updated_at": "2025-01-01T00:00:00Z", + }, + { + "index_item_id": "scoped_assertion:ska_1", + "row_kind": "scoped_assertion", + "relation_id": None, + "assertion_id": "ska_1", + "subject_concept_id": "#V#x", + "predicate": "hasNote", + "text_value_id": None, + "lang": "en-NZ", + "preview_length": 10, + "updated_at": "2025-01-02T00:00:00Z", } ], - "total": 1, - "scanned": 1, + "total": 2, + "scanned": 2, "limit": 10, "offset": 0, "scan_limit": 5000, @@ -47,6 +62,16 @@ def test_rag_list_indexed_supports_vontology_text_relations(monkeypatch): assert item["item_kind"] == "vontology_text_relation" assert item["source_system"] == "mongo.text_relations" assert item["session_id"] == "r1" + assert item["index_item_id"] == "text_relation:r1" + assert item["row_kind"] == "base_text_relation" + + scoped_item = result["items"][1] + assert scoped_item["session_id"] == "ska_1" + assert scoped_item["index_item_id"] == "scoped_assertion:ska_1" + assert scoped_item["relation_id"] is None + assert scoped_item["assertion_id"] == "ska_1" + assert scoped_item["row_kind"] == "scoped_assertion" + assert scoped_item["source_system"] == "mongo.scoped_knowledge_assertions" def test_rag_get_item_supports_vontology_text_relations(monkeypatch): @@ -59,11 +84,14 @@ def test_rag_get_item_supports_vontology_text_relations(monkeypatch): class _Doc: def __init__(self): + self.doc_id = "text_relation:r1" self.text = ( "Concept: #V#x\nPredicate: hasDescription\nLanguage: en-NZ\n\nHello" ) self.metadata = { + "row_kind": "base_text_relation", "relation_id": "r1", + "assertion_id": None, "subject_concept_id": "#V#x", "predicate": "hasDescription", "lang": "en-NZ", @@ -84,3 +112,6 @@ def __init__(self): assert result["collection"] == "vontology_text_relations" assert result["provenance"]["item_kind"] == "rag_text_relation_item" assert "Hello" in result["preview"] + assert result["index_item_id"] == "text_relation:r1" + assert result["row_kind"] == "base_text_relation" + assert result["assertion_id"] is None diff --git a/tests/backend/test_reindex_text_relations_cli.py b/tests/backend/test_reindex_text_relations_cli.py index d6350e69c..bf7d8c57a 100644 --- a/tests/backend/test_reindex_text_relations_cli.py +++ b/tests/backend/test_reindex_text_relations_cli.py @@ -58,7 +58,7 @@ def _boom(): monkeypatch.setattr(mod, "get_rag_service", _boom) monkeypatch.setattr(mod.TextRelationsRepository, "count_documents", lambda _f: 0) monkeypatch.setattr( - mod, "collect_text_relation_docs_for_namespace", lambda **_k: [] + mod, "iter_text_relation_docs_for_namespace", lambda **_k: iter(()) ) args = ReindexArgs( @@ -95,7 +95,9 @@ def upsert_documents(self, docs, namespace: str): doc_id="text_relation:1", text="hello", metadata={"k": "v"} ) monkeypatch.setattr( - mod, "collect_text_relation_docs_for_namespace", lambda **_k: [doc] + mod, + "iter_text_relation_docs_for_namespace", + lambda **_k: iter([doc]), ) args = ReindexArgs( @@ -114,3 +116,164 @@ def upsert_documents(self, docs, namespace: str): assert calls["upserts"] == 1 assert calls["namespace"] == "#V#u@#V#org" assert calls["count"] == 1 + + +def test_run_reindex_scans_combined_source_when_base_count_is_zero( + monkeypatch: pytest.MonkeyPatch, +): + from src.utilities import reindex_text_relations as mod + + monkeypatch.setattr(mod.TextRelationsRepository, "count_documents", lambda _f: 0) + calls = [] + scoped_doc = TextRelationRagDoc( + doc_id="scoped_assertion:ska_only", + text="scoped", + metadata={"type": "scoped_knowledge_assertion"}, + ) + + def _iterate(**kwargs): + calls.append( + (kwargs["limit"], kwargs["source_batch_size"]) + ) + return iter([scoped_doc]) + + monkeypatch.setattr( + mod, + "iter_text_relation_docs_for_namespace", + _iterate, + ) + + args = ReindexArgs( + namespace="#V#u@#V#org", + predicates=None, + languages=None, + concept_ids=None, + updated_since=None, + dry_run=True, + page_size=1, + batch_size=10, + scan_limit=0, + ) + + assert run_reindex(args) == 0 + assert calls == [(None, 1)] + + +def test_run_reindex_consumes_one_iterator_and_batches_writes( + monkeypatch: pytest.MonkeyPatch, +): + from src.utilities import reindex_text_relations as mod + + calls = [] + upsert_sizes = [] + + class _FakeRag: + def upsert_documents(self, docs, namespace: str): + upsert_sizes.append(len(docs)) + return len(docs), 0 + + docs = [ + TextRelationRagDoc( + doc_id=f"text_relation:{index}", + text=f"text {index}", + metadata={}, + ) + for index in range(5) + ] + + def _iterate(**kwargs): + calls.append(kwargs) + return iter(docs) + + monkeypatch.setattr(mod, "get_rag_service", lambda: _FakeRag()) + monkeypatch.setattr( + mod.TextRelationsRepository, + "count_documents", + lambda _filter: 5, + ) + monkeypatch.setattr( + mod, + "iter_text_relation_docs_for_namespace", + _iterate, + ) + args = ReindexArgs( + namespace="#V#u@#V#org", + predicates=None, + languages=None, + concept_ids=None, + updated_since=None, + dry_run=False, + page_size=1, + batch_size=2, + scan_limit=0, + ) + + assert run_reindex(args) == 0 + assert len(calls) == 1 + assert calls[0]["limit"] is None + assert calls[0]["source_batch_size"] == 1 + assert upsert_sizes == [2, 2, 1] + + +def test_run_reindex_flushes_stale_only_batches_incrementally( + monkeypatch: pytest.MonkeyPatch, +): + from src.utilities import reindex_text_relations as mod + + delete_batches = [] + + class _FakeRag: + def upsert_documents(self, docs, namespace: str): + raise AssertionError("stale-only scan must not upsert") + + def delete_documents(self, ids, namespace: str): + batch = list(ids) + delete_batches.append(batch) + return len(batch) + + def _iterate(**kwargs): + stale_sink = kwargs["stale_doc_sink"] + + def _stale_only_source(): + stale_sink( + [ + "text_relation:r0", + "text_relation:r1", + "text_relation:r2", + ] + ) + stale_sink(["scoped_assertion:ska_retracted"]) + return + yield # pragma: no cover + + return _stale_only_source() + + monkeypatch.setattr(mod, "get_rag_service", lambda: _FakeRag()) + monkeypatch.setattr( + mod.TextRelationsRepository, + "count_documents", + lambda _filter: 3, + ) + monkeypatch.setattr( + mod, + "iter_text_relation_docs_for_namespace", + _iterate, + ) + args = ReindexArgs( + namespace="#V#u@#V#org", + predicates=None, + languages=None, + concept_ids=None, + updated_since=None, + dry_run=False, + page_size=3, + batch_size=2, + scan_limit=0, + ) + + assert run_reindex(args) == 0 + assert delete_batches == [ + ["text_relation:r0", "text_relation:r1"], + ["text_relation:r2"], + ["scoped_assertion:ska_retracted"], + ] diff --git a/tests/backend/test_scoped_assertion_service.py b/tests/backend/test_scoped_assertion_service.py index 5ca7e1b06..4fb0ea21d 100644 --- a/tests/backend/test_scoped_assertion_service.py +++ b/tests/backend/test_scoped_assertion_service.py @@ -1,5 +1,8 @@ from __future__ import annotations +from datetime import datetime, timezone +import json + import mongomock import pytest @@ -8,6 +11,40 @@ def _collection(): return mongomock.MongoClient()["von_test"]["scoped_knowledge_assertions"] +def _stored_text_assertion( + *, + assertion_id: str, + subject_concept_id: str, + audience_key: str = "org:#V#trusted_org", +) -> dict: + now = datetime.now(timezone.utc) + return { + "schema_version": "scoped_knowledge_assertion.v1", + "assertion_id": assertion_id, + "subject_concept_id": subject_concept_id, + "predicate": "#V#has_description", + "object_kind": "text", + "object_text": { + "text": f"Text for {assertion_id}", + "language": "en-NZ", + }, + "object_concept_id": None, + "scope": { + "mode": "organisation", + "user_concept_id": "#V#author", + "organisation_concept_id": "#V#trusted_org", + "audience_keys": [audience_key], + }, + "provenance": { + "asserted_by_user_concept_id": "#V#author", + }, + "canonical_publication": False, + "status": "asserted", + "created_at": now, + "updated_at": now, + } + + def test_visible_global_subject_accepts_user_scoped_text_assertion(monkeypatch): from src.backend.services import scoped_assertion_service as service @@ -56,6 +93,67 @@ def test_visible_global_subject_accepts_user_scoped_text_assertion(monkeypatch): assert read_back["provenance"]["turn_id"] == "turn-1" +def test_org_members_keep_distinct_immutable_assertion_provenance(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + + first = service.upsert_scoped_assertion( + subject_concept_id="#V#shared_subject", + predicate="hasDescription", + target_text="The same organisation-visible claim.", + scope_mode="organisation", + evidence={"source": "first-member"}, + acting_user_concept_id="#V#member_one", + organisation_concept_id="#V#trusted_org", + turn_id="member-one-turn", + ) + second = service.upsert_scoped_assertion( + subject_concept_id="#V#shared_subject", + predicate="hasDescription", + target_text="The same organisation-visible claim.", + scope_mode="organisation", + evidence={"source": "second-member"}, + acting_user_concept_id="#V#member_two", + organisation_concept_id="#V#trusted_org", + turn_id="member-two-turn", + ) + + assert first["assertion_id"] != second["assertion_id"] + assert collection.count_documents({}) == 2 + assert ( + first["assertion"]["provenance"]["asserted_by_user_concept_id"] + == "#V#member_one" + ) + assert ( + second["assertion"]["provenance"]["asserted_by_user_concept_id"] + == "#V#member_two" + ) + + repeated = service.upsert_scoped_assertion( + subject_concept_id="#V#shared_subject", + predicate="hasDescription", + target_text="The same organisation-visible claim.", + scope_mode="organisation", + evidence={"source": "replacement-attempt"}, + acting_user_concept_id="#V#member_one", + organisation_concept_id="#V#trusted_org", + turn_id="later-turn", + ) + assert repeated["changed"] is False + assert repeated["assertion_id"] == first["assertion_id"] + assert repeated["assertion"]["provenance"]["turn_id"] == "member-one-turn" + assert repeated["assertion"]["provenance"]["evidence"] == { + "source": "first-member" + } + + def test_scoped_assertions_do_not_cross_actor_or_organisation(monkeypatch): from src.backend.services import scoped_assertion_service as service @@ -92,6 +190,356 @@ def test_scoped_assertions_do_not_cross_actor_or_organisation(monkeypatch): assert other_org == [] +def test_unfiltered_reads_recheck_subject_and_concept_target_visibility(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + visible_concepts = { + "#V#subject", + "#V#target", + "#V#hasDescription", + "#V#related_to", + } + monkeypatch.setattr( + service, + "can_access_concept", + lambda concept_id: concept_id in visible_concepts, + ) + monkeypatch.setattr( + service, + "filter_accessible_concept_ids", + lambda concept_ids: { + concept_id + for concept_id in concept_ids + if concept_id in visible_concepts + }, + ) + monkeypatch.setattr( + service, + "validate_predicate_concept", + lambda _predicate: (True, None, None), + ) + service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="A visible text claim.", + scope_mode="organisation", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="#V#related_to", + target_concept_id="#V#target", + scope_mode="organisation", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + initially_visible = service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + assert len(initially_visible) == 2 + + visible_concepts.remove("#V#related_to") + after_predicate_revocation = service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + assert [item["object_kind"] for item in after_predicate_revocation] == [ + "text" + ] + visible_concepts.add("#V#related_to") + + visible_concepts.remove("#V#target") + target_revocation_page = service.list_visible_scoped_assertions_page( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + assert [item["object_kind"] for item in target_revocation_page["items"]] == [ + "text" + ] + assert target_revocation_page["visibility_filtered"] is False + assert target_revocation_page["counts_are_lower_bounds"] is False + + visible_concepts.remove("#V#hasDescription") + assert ( + service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + == [] + ) + + visible_concepts.remove("#V#subject") + after_subject_revocation = service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + assert after_subject_revocation == [] + + +def test_explicit_read_actor_must_match_ambient_trusted_actor(monkeypatch): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service as service + + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + _collection, + ) + with override_current_actor("#V#member", "#V#organisation_a"): + with pytest.raises(PermissionError, match="organisation audience"): + service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#organisation_b", + ) + with pytest.raises(PermissionError, match="user audience"): + service.list_visible_scoped_assertions( + user_concept_id="#V#other_member", + organisation_concept_id="#V#organisation_a", + ) + # A user whose effective organisation has been revoked cannot revive the + # old audience by supplying its identifier explicitly. + with override_current_actor("#V#member", None): + with pytest.raises(PermissionError, match="organisation audience"): + service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#organisation_a", + ) + + +def test_explicit_read_actor_binds_current_visibility_checks(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + collection.insert_one( + _stored_text_assertion( + assertion_id="ska_one", + subject_concept_id="#V#subject", + ) + ) + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + observed_actors = [] + + def _filter_accessible(concept_ids): + observed_actors.append( + ( + service.get_effective_user_concept_id(), + service.get_effective_organisation_concept_id(), + ) + ) + return set(concept_ids) + + monkeypatch.setattr( + service, + "filter_accessible_concept_ids", + _filter_accessible, + ) + assertions = service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + assert len(assertions) == 1 + assert observed_actors + assert set(observed_actors) == { + ("#V#member", "#V#trusted_org"), + } + + +def test_per_subject_page_is_one_bounded_non_starving_aggregate(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + subjects = ["#V#subject_a", "#V#subject_b"] + documents = [ + _stored_text_assertion( + assertion_id=f"ska_{subject[-1]}_{index}", + subject_concept_id=subject, + ) + for subject in subjects + for index in range(3) + ] + + class _AggregateCollection: + name = "scoped_knowledge_assertions" + + def __init__(self): + self.pipelines = [] + + def aggregate(self, pipeline): + self.pipelines.append(pipeline) + return iter(documents) + + collection = _AggregateCollection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + + page = service.list_visible_scoped_assertions_page( + subject_concept_ids=subjects, + object_kind="text", + limit_per_subject=2, + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + assert len(collection.pipelines) == 1 + pipeline = collection.pipelines[0] + union_stages = [stage["$unionWith"] for stage in pipeline if "$unionWith" in stage] + assert len(union_stages) == 1 + assert pipeline[2] == {"$limit": 5001} + assert union_stages[0]["pipeline"][2] == {"$limit": 5001} + assert all("$skip" not in stage for stage in pipeline) + assert all( + "$skip" not in stage + for stage in union_stages[0]["pipeline"] + ) + assert page["returned"] == 4 + assert page["has_more"] is True + assert page["truncated"] is True + assert page["counts_are_lower_bounds"] is True + for subject in subjects: + subject_page = page["per_subject"][subject] + assert subject_page["returned"] == 2 + assert subject_page["has_more"] is True + assert subject_page["next_offset"] == 2 + assert all( + item["subject_concept_id"] == subject + for item in subject_page["items"] + ) + + +def test_hidden_candidates_do_not_change_visible_paging_or_diagnostics( + monkeypatch, +): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + newest_hidden = _stored_text_assertion( + assertion_id="ska_hidden", + subject_concept_id="#V#hidden", + ) + first_visible = _stored_text_assertion( + assertion_id="ska_visible_1", + subject_concept_id="#V#visible_1", + ) + second_visible = _stored_text_assertion( + assertion_id="ska_visible_2", + subject_concept_id="#V#visible_2", + ) + newest_hidden["updated_at"] = datetime(2026, 7, 29, 12, tzinfo=timezone.utc) + first_visible["updated_at"] = datetime( + 2026, 7, 29, 11, tzinfo=timezone.utc + ) + second_visible["updated_at"] = datetime( + 2026, 7, 29, 10, tzinfo=timezone.utc + ) + collection.insert_many([newest_hidden, first_visible, second_visible]) + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + visibility_batches = [] + + def _filter_accessible(concept_ids): + batch = set(concept_ids) + if batch: + visibility_batches.append(batch) + return { + concept_id + for concept_id in batch + if concept_id != "#V#hidden" + } + + monkeypatch.setattr( + service, + "filter_accessible_concept_ids", + _filter_accessible, + ) + + first_page = service.list_visible_scoped_assertions_page( + limit=1, + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + second_page = service.list_visible_scoped_assertions_page( + limit=1, + offset=1, + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + assert [item["assertion_id"] for item in first_page["items"]] == [ + "ska_visible_1" + ] + assert first_page["has_more"] is True + assert first_page["next_offset"] == 1 + assert first_page["visibility_filtered"] is False + assert first_page["counts_are_lower_bounds"] is True + assert [item["assertion_id"] for item in second_page["items"]] == [ + "ska_visible_2" + ] + assert second_page["has_more"] is False + assert second_page["next_offset"] is None + assert second_page["visibility_filtered"] is False + assert second_page["counts_are_lower_bounds"] is False + assert len(visibility_batches) == 2 + assert all("#V#hidden" in batch for batch in visibility_batches) + + +def test_core_predicate_filters_match_storage_and_concept_id_forms( + monkeypatch, +): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + stored = _stored_text_assertion( + assertion_id="ska_core_predicate", + subject_concept_id="#V#visible", + ) + stored["predicate"] = "hasNote" + collection.insert_one(stored) + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr( + service, + "filter_accessible_concept_ids", + lambda concept_ids: set(concept_ids), + ) + + for predicate in ("hasNote", "#V#hasNote"): + page = service.list_visible_scoped_assertions_page( + predicates=[predicate], + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + assert [row["assertion_id"] for row in page["items"]] == [ + "ska_core_predicate" + ] + + def test_service_rejects_namespace_that_disagrees_with_trusted_actor(monkeypatch): from src.backend.services import scoped_assertion_service as service @@ -114,6 +562,317 @@ def test_service_rejects_namespace_that_disagrees_with_trusted_actor(monkeypatch ) +def test_write_actor_cannot_conflict_with_ambient_trusted_actor(monkeypatch): + from src.backend.security.access_control import override_current_actor + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + + def _unexpected_access(_concept_id): + raise AssertionError("actor conflict must fail before concept access") + + monkeypatch.setattr(service, "can_access_concept", _unexpected_access) + with override_current_actor("#V#member", "#V#organisation_a"): + with pytest.raises(PermissionError, match="acting user"): + service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="Conflicting user.", + acting_user_concept_id="#V#other_member", + organisation_concept_id="#V#organisation_a", + ) + with pytest.raises(PermissionError, match="organisation"): + service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="Conflicting organisation.", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#organisation_b", + ) + assert collection.count_documents({}) == 0 + + +def test_predicate_visibility_precedes_concept_predicate_validation(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + events = [] + + def _can_access(concept_id): + events.append(("access", concept_id)) + return concept_id != "#V#hidden_predicate" + + def _validate(predicate): + events.append(("validate", predicate)) + return True, None, None + + monkeypatch.setattr(service, "can_access_concept", _can_access) + monkeypatch.setattr(service, "validate_predicate_concept", _validate) + + with pytest.raises(PermissionError, match="predicate concept"): + service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="#V#hidden_predicate", + target_concept_id="#V#target", + acting_user_concept_id="#V#member", + ) + + assert ("access", "#V#hidden_predicate") in events + assert not any(event[0] == "validate" for event in events) + assert collection.count_documents({}) == 0 + + +def test_user_scope_is_org_independent_and_replay_preserves_origin(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + + first = service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="User-enduring claim.", + scope_mode="user", + evidence={"source": "first-context"}, + acting_user_concept_id="#V#member", + organisation_concept_id="#V#organisation_a", + namespace="#V#member@organisation_a", + turn_id="first-turn", + ) + replay = service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="User-enduring claim.", + scope_mode="user", + evidence={"source": "replacement-attempt"}, + acting_user_concept_id="#V#member", + organisation_concept_id="#V#organisation_b", + namespace="#V#member@organisation_b", + turn_id="replay-turn", + ) + + assert first["changed"] is True + assert replay["changed"] is False + assert replay["assertion_id"] == first["assertion_id"] + assert collection.count_documents({}) == 1 + assertion = replay["canonical_read_back"] + assert assertion["scope"]["organisation_concept_id"] is None + assert assertion["scope"]["namespace"] == service.resolve_canonical_namespace( + None, + "#V#member", + None, + ) + assert assertion["provenance"]["organisation_concept_id"] == "#V#organisation_a" + assert assertion["provenance"]["namespace"] == "#V#member@organisation_a" + assert assertion["provenance"]["turn_id"] == "first-turn" + assert assertion["provenance"]["evidence"] == { + "source": "first-context" + } + assert replay["canonical_read_back"] == first["canonical_read_back"] + + retracted = service.retract_scoped_assertion( + assertion_id=first["assertion_id"], + acting_user_concept_id="#V#member", + organisation_concept_id="#V#organisation_b", + namespace="#V#member@organisation_b", + ) + assert retracted["changed"] is True + assert retracted["assertion"]["status"] == "retracted" + + +def test_atomic_creation_reports_changed_once_under_concurrency(monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + worker_count = 8 + barrier = Barrier(worker_count) + + def _write(index): + barrier.wait() + return service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="One concurrently asserted claim.", + scope_mode="organisation", + evidence={"worker": index}, + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + turn_id=f"turn-{index}", + ) + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + receipts = list(executor.map(_write, range(worker_count))) + + assert sum(receipt["changed"] is True for receipt in receipts) == 1 + assert collection.count_documents({}) == 1 + assert collection.find_one({})["_id"] == receipts[0]["assertion_id"] + canonical_ids = { + receipt["canonical_read_back"]["assertion_id"] for receipt in receipts + } + assert canonical_ids == {receipts[0]["assertion_id"]} + original_provenance = collection.find_one( + {"assertion_id": receipts[0]["assertion_id"]} + )["provenance"] + + service.retract_scoped_assertion( + assertion_id=receipts[0]["assertion_id"], + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reactivation_receipts = list( + executor.map(_write, range(worker_count)) + ) + + assert ( + sum(receipt["changed"] is True for receipt in reactivation_receipts) + == 1 + ) + reactivated = collection.find_one( + {"assertion_id": receipts[0]["assertion_id"]} + ) + assert reactivated["status"] == "asserted" + assert reactivated["provenance"] == original_provenance + assert "retracted_at" not in reactivated + assert len(reactivated["retraction_history"]) == 1 + assert reactivated["reassertion_provenance"][ + "reasserted_by_user_concept_id" + ] == "#V#member" + for receipt in reactivation_receipts: + json.dumps(receipt["canonical_read_back"]) + + +def test_original_author_can_idempotently_retract_after_visibility_loss(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + receipt = service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="Retractable claim.", + scope_mode="organisation", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + def _visibility_must_not_be_consulted(_concept_id): + raise AssertionError("retraction must survive concept visibility loss") + + monkeypatch.setattr( + service, + "can_access_concept", + _visibility_must_not_be_consulted, + ) + first = service.retract_scoped_assertion( + assertion_id=receipt["assertion_id"], + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + repeated = service.retract_scoped_assertion( + assertion_id=receipt["assertion_id"], + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + assert first["success"] is True + assert first["effect_status"] == "succeeded" + assert first["changed"] is True + assert first["assertion"]["status"] == "retracted" + assert first["assertion"]["retracted_at"] + assert first["assertion"]["retraction_provenance"] == { + "retracted_by_user_concept_id": "#V#member", + "organisation_concept_id": "#V#trusted_org", + "namespace": "#V#member@trusted_org", + "capability_name": "retract_scoped_assertion", + } + assert repeated["changed"] is False + assert repeated["canonical_read_back"] == first["canonical_read_back"] + assert ( + service.list_visible_scoped_assertions( + user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + == [] + ) + + +def test_retraction_denial_does_not_reveal_existence(monkeypatch): + from src.backend.services import scoped_assertion_service as service + + collection = _collection() + monkeypatch.setattr( + service, + "get_scoped_knowledge_assertions_collection", + lambda: collection, + ) + monkeypatch.setattr(service, "can_access_concept", lambda _concept_id: True) + receipt = service.upsert_scoped_assertion( + subject_concept_id="#V#subject", + predicate="hasDescription", + target_text="Owned claim.", + scope_mode="organisation", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + with pytest.raises(PermissionError) as wrong_author: + service.retract_scoped_assertion( + assertion_id=receipt["assertion_id"], + acting_user_concept_id="#V#other_member", + organisation_concept_id="#V#trusted_org", + ) + with pytest.raises(PermissionError) as wrong_org: + service.retract_scoped_assertion( + assertion_id=receipt["assertion_id"], + acting_user_concept_id="#V#member", + organisation_concept_id="#V#other_org", + ) + with pytest.raises(PermissionError) as missing: + service.retract_scoped_assertion( + assertion_id="ska_00000000000000000000000000000000", + acting_user_concept_id="#V#member", + organisation_concept_id="#V#trusted_org", + ) + + expected = "scoped assertion is unavailable to the current actor" + assert str(wrong_author.value) == expected + assert str(wrong_org.value) == expected + assert str(missing.value) == expected + assert collection.find_one( + {"assertion_id": receipt["assertion_id"]} + )["status"] == "asserted" + + def test_concept_target_assertion_is_retrievable_from_either_argument(monkeypatch): from src.backend.services import scoped_assertion_service as service @@ -199,7 +958,7 @@ def test_trusted_tool_payload_replaces_model_supplied_scope(): assert payload["canonical_publication"] is False -def test_generic_text_read_merges_visible_scoped_assertions(monkeypatch): +def test_actor_effective_text_read_merges_visible_scoped_assertions(monkeypatch): from src.backend.security.access_control import override_current_actor from src.backend.services import scoped_assertion_service as scoped_service from src.backend.services import text_value_service @@ -239,10 +998,13 @@ def test_generic_text_read_merges_visible_scoped_assertions(monkeypatch): rows = text_value_service.get_texts_for_concept( "#V#gillian_dobbie", predicate="hasDescription", + context_view="actor_effective", ) assert len(rows) == 1 assert rows[0]["text"] == "Scoped programme context." assert rows[0]["assertion_id"].startswith("ska_") + assert rows[0]["relation_id"] is None + assert rows[0]["row_kind"] == "scoped_assertion" assert rows[0]["canonical_publication"] is False assert rows[0]["storage_surface"] == "scoped_knowledge_assertions"