From 07c15c9898e6b191673dce5a06dc19aae4614926 Mon Sep 17 00:00:00 2001 From: witbrock Date: Fri, 3 Jul 2026 12:53:08 -0400 Subject: [PATCH] JVNAUTOSCI-2561 narrow replay Gmail blocker oracle --- AGENTS.md | 1 + ...n_authenticated_browser_workflow_replay.py | 134 ++++++++++++++--- ...n_authenticated_browser_workflow_replay.py | 141 ++++++++++++++++++ 3 files changed, 259 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 49c5e9c6a..990874688 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,6 +112,7 @@ For frontend/browser user-view validation practice, also see 38. Domain-handler-first implementation is a warning sign, not a plan. If the first substantial edit for a workflow/prompt/KB-authoritative task is inside a domain-specific Python workflow handler, service, registry entry, fallback table, or verifier, pause before continuing and write down why the behaviour cannot be expressed by an existing or newly added generic VWL/Vontology/tool-language surface. If that explanation is "the workflow language is missing a small reusable capability", add that capability first and then express the behaviour in the workflow. 39. Do not use a legacy Python workflow handler as a temporary staging area for authored behaviour while intending to "move it to workflow later". That pattern usually becomes the implementation. Prototype or materialise the workflow/prompt/Vontology artefact first; add Python only for the named reusable primitive, validation, telemetry, or canonical tool bridge that the artefact needs. 40. When a task claims to repair a live user-visible turn, workflow failure, or motivating prompt family, closure requires replay of the exact triggering prompt or conversation/session reference on the real user-facing path, plus a small nearby prompt family where appropriate. Unit tests, materialised artefacts, synthetic helper calls, or a single nearby pass are not sufficient. The final claim must reconcile the user-visible answer, selector/discovery/dispatch telemetry, workflow/tool execution state, Thinking-card or debug surfaces, and read-back or typed failure evidence. Treat partial or contradictory diagnostic content as hypotheses, not truth. +41. After any replay used as validation evidence, perform a path analysis whether the replay passed, failed, timed out, or returned a typed blocker. Confirm how the candidate set was generated, what the selector actually saw, what it chose and why, which workflow or tool path executed, what evidence was written or read back, and whether the final answer followed from that evidence. Distinguish a legitimate success path from a lucky or nearby answer, a fallback that merely sounded plausible, a real downstream blocker, or a harness oracle that reported the wrong thing. Record path-analysis discrepancies in Jira or durable docs before closing the task, so future agents do not patch around symptoms or bless behaviour that only feels kind of right. ### 3.1 Opportunity-preservation gate diff --git a/scripts/run_authenticated_browser_workflow_replay.py b/scripts/run_authenticated_browser_workflow_replay.py index d657853a5..4faaa0471 100644 --- a/scripts/run_authenticated_browser_workflow_replay.py +++ b/scripts/run_authenticated_browser_workflow_replay.py @@ -1090,24 +1090,124 @@ def _contract_ids(facts: Sequence[Mapping[str, Any]]) -> set[str]: } -def _contains_gmail_oauth_blocker_text(value: Any) -> bool: - text = json.dumps(value, ensure_ascii=True, sort_keys=True).lower() - gmailish = any(marker in text for marker in ("gmail", "google mail")) - authish = any( - marker in text - for marker in ( - "oauth", - "token", - "credential", - "authoris", - "authoriz", - "profile", - "not configured", - "not authenticated", - "not connected", +_GMAIL_OAUTH_AUTH_MARKERS = ( + "oauth", + "token", + "credential", + "authoris", + "authoriz", + "profile", +) +_GMAIL_OAUTH_FAILURE_MARKERS = ( + "denied", + "disabled", + "disconnected", + "expired", + "failed", + "failure", + "invalid", + "missing", + "not authenticated", + "not configured", + "not connected", + "not ready", + "reauth", + "re-auth", + "revoked", + "unavailable", +) +_GMAIL_CONTEXT_KEYS = ( + "gmail", + "gmail_preflight", + "gmail_profile", + "requested_gmail_profile", +) +_GMAIL_IDENTIFIER_KEYS = { + "action_id", + "concept_id", + "contract_id", + "fact_id", + "id", + "selected_workflow_id", + "state_id", + "workflow_id", +} +_GMAIL_OAUTH_NON_EVIDENCE_KEYS = { + "available_tools", + "context_message", + "context_messages", + "llm_prompt", + "llm_request", + "messages", + "prompt", + "prompt_preview", + "request", + "selector_prompt", + "system_prompt", + "tool_specs", +} + + +def _text_mentions_gmail(value: str) -> bool: + text = value.lower() + return "gmail" in text or "google mail" in text + + +def _text_mentions_gmail_oauth_failure(value: str, *, gmail_context: bool) -> bool: + text = value.lower() + gmailish = gmail_context or _text_mentions_gmail(text) + if not gmailish: + return False + authish = any(marker in text for marker in _GMAIL_OAUTH_AUTH_MARKERS) + failureish = any(marker in text for marker in _GMAIL_OAUTH_FAILURE_MARKERS) + return authish and failureish + + +def _mapping_has_gmail_context(value: Mapping[str, Any]) -> bool: + for raw_key, raw_item in value.items(): + key = _safe_text(raw_key).lower() + if key in _GMAIL_IDENTIFIER_KEYS: + continue + if any(marker in key for marker in _GMAIL_CONTEXT_KEYS): + return True + if isinstance(raw_item, str) and _text_mentions_gmail(raw_item): + return True + return False + + +def _contains_gmail_oauth_blocker_text( + value: Any, + *, + gmail_context: bool = False, +) -> bool: + if isinstance(value, str): + return _text_mentions_gmail_oauth_failure(value, gmail_context=gmail_context) + if isinstance(value, Mapping): + local_gmail_context = gmail_context or _mapping_has_gmail_context(value) + for raw_key, item in value.items(): + key = _safe_text(raw_key).lower() + if key in _GMAIL_OAUTH_NON_EVIDENCE_KEYS: + continue + child_gmail_context = local_gmail_context + if key in _GMAIL_IDENTIFIER_KEYS: + child_gmail_context = gmail_context + elif any(marker in key for marker in _GMAIL_CONTEXT_KEYS): + child_gmail_context = True + if _contains_gmail_oauth_blocker_text( + item, + gmail_context=child_gmail_context, + ): + return True + return False + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return any( + _contains_gmail_oauth_blocker_text( + item, + gmail_context=gmail_context, + ) + for item in value ) - ) - return gmailish and authish + return False def _selector_route_evidence_present( diff --git a/tests/test_run_authenticated_browser_workflow_replay.py b/tests/test_run_authenticated_browser_workflow_replay.py index 3afdead81..6c8f5e548 100644 --- a/tests/test_run_authenticated_browser_workflow_replay.py +++ b/tests/test_run_authenticated_browser_workflow_replay.py @@ -406,6 +406,147 @@ def test_classify_replay_passes_when_expected_workflow_and_projection_seen() -> assert analysis["missing_contract_ids"] == [] +def test_classify_replay_ignores_gmail_arxiv_progress_fact_names() -> None: + case = replay.ReplayCase( + case_id="arxiv-paper-representation", + prompt="Ingest https://arxiv.org/abs/2406.15341", + expected_workflow_id="#V#arxiv_paper_representation_workflow", + expected_progress_fact_ids=(), + expected_contract_ids=(), + ) + + analysis = replay.classify_replay( + case=case, + auth_login={"success": True}, + auth_status={"authenticated": True}, + gmail_preflight={}, + task_evidence={ + "last_task_status": {"status": "completed"}, + "task_result": { + "result": { + "response_text": ( + "Verified workflow experience profile concept: " + "#V#workflow_llm_experience_profile_v_conversation_turn_execution_workflow_gpt_5_4_nano." + ) + } + }, + "thinking_card_progress_facts": [ + { + "fact_id": "arxiv_paper_title", + "contract_id": "#V#gmail_arxiv_paper_title_progress_fact", + }, + { + "fact_id": "arxiv_paper_id", + "contract_id": "#V#gmail_arxiv_metadata_arxiv_id_progress_fact", + }, + ], + }, + selected_workflow_ids=["#V#arxiv_paper_representation_workflow"], + observed_workflow_ids=["#V#arxiv_paper_representation_workflow"], + selector_diagnostics=[], + progress_facts=[], + ) + + assert analysis["verdict"] == "pass" + assert analysis["blocker"] is None + + +def test_classify_replay_ignores_gmail_auth_text_in_llm_prompts() -> None: + case = replay.ReplayCase( + case_id="arxiv-paper-representation", + prompt="Ingest https://arxiv.org/abs/2406.15341", + expected_workflow_id="#V#arxiv_paper_representation_workflow", + expected_progress_fact_ids=(), + expected_contract_ids=(), + ) + + analysis = replay.classify_replay( + case=case, + auth_login={"success": True}, + auth_status={"authenticated": True}, + gmail_preflight={}, + task_evidence={ + "last_task_status": {"status": "completed"}, + "progress_snapshots": [ + { + "diagnostic_events": [ + { + "llm_request": { + "context_messages": [ + { + "content": { + "text": ( + "AUTHENTICATION STATUS: Authenticated. " + "Use Gmail tools only when the user asks " + "for mail; if not authenticated, explain " + "that OAuth tokens are required." + ) + } + } + ] + } + } + ], + "stage_diagnostics": [ + { + "latest_llm_exchange": { + "prompt_preview": { + "text": ( + "Prompt text says Gmail OAuth tokens may " + "be required if mail is not connected." + ) + } + } + } + ], + } + ], + }, + selected_workflow_ids=["#V#arxiv_paper_representation_workflow"], + observed_workflow_ids=["#V#arxiv_paper_representation_workflow"], + selector_diagnostics=[], + progress_facts=[], + ) + + assert analysis["verdict"] == "pass" + assert analysis["blocker"] is None + + +def test_classify_replay_reports_explicit_gmail_oauth_failure_text() -> None: + case = replay.ReplayCase( + case_id="generic-gmail-posthoc-failure", + prompt="Inspect Gmail", + expected_workflow_id="#V#tool_calling_workflow", + expected_progress_fact_ids=(), + expected_contract_ids=(), + ) + + analysis = replay.classify_replay( + case=case, + auth_login={"success": True}, + auth_status={"authenticated": True}, + gmail_preflight={}, + task_evidence={ + "last_task_status": {"status": "completed"}, + "task_result": { + "result": { + "response_text": ( + "Gmail OAuth token is missing; the Gmail profile is not " + "authenticated." + ) + } + }, + }, + selected_workflow_ids=["#V#tool_calling_workflow"], + observed_workflow_ids=["#V#tool_calling_workflow"], + selector_diagnostics=[], + progress_facts=[], + ) + + assert analysis["verdict"] == "blocked" + assert analysis["blocker"]["type"] == "gmail_oauth_or_profile_blocker" + + def test_classify_replay_reports_selector_blocker_before_running_timeout() -> None: analysis = replay.classify_replay( case=replay.GMAIL_ARXIV_REPLAY_CASE,