From 1ec2f04a67edb625c532e946b37e3b4ef77c3fe6 Mon Sep 17 00:00:00 2001 From: DavidHLP Date: Sat, 10 Oct 2026 20:26:44 -0700 Subject: [PATCH 1/3] test(agent): cover cited refusal behavior overlap --- .../agent/tests/test_answer_evaluation.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/services/agent/tests/test_answer_evaluation.py b/services/agent/tests/test_answer_evaluation.py index e78475a73..78f89da58 100644 --- a/services/agent/tests/test_answer_evaluation.py +++ b/services/agent/tests/test_answer_evaluation.py @@ -316,6 +316,27 @@ def test_a_behavior_mismatch_is_recorded_not_hidden() -> None: assert row.observed_behavior == "cite" assert row.behavior_match is False assert row.citation_support == "unsupported" + + +def test_a_cited_refusal_keeps_refusal_behavior() -> None: + text = "The cited status fragment cannot identify a code line, so I cannot name one." + answers = {"dev-01": json.dumps({"text": text, "citations": ["sample-status-only:v1:1"]})} + judgements = { + "dev-01": '{"citation_support": true, "answer_completed": true, "observed_behavior": "refuse"}' + } + rows, model = _run([_case(expected="refuse")], answers, judgements) + + row = rows[0] + assert row.answer_text == text + assert row.citations == ("sample-status-only:v1:1",) + assert row.citation_support == "supported" + assert row.answer_completion == "completed" + assert row.observed_behavior == "refuse" + assert row.behavior_match is True + prompt = model.prompts[1] + assert "refuse > clarify > no_evidence > cite" in prompt + assert "A citation never overrides a higher-priority behavior" in prompt + assert json.loads(prompt.rsplit("\nANSWER_JSON ", 1)[1]) == text assert row.answer_completion == "incomplete" From 8bc31e9b9514018a960d30747c7f9d9355ad4052 Mon Sep 17 00:00:00 2001 From: DavidHLP Date: Sat, 10 Oct 2026 20:27:51 -0700 Subject: [PATCH 2/3] fix(agent): make judging behavior categories mutually exclusive --- docs/DEVELOPMENT.md | 6 +++++- services/agent/data/repository_corpus_manifest.json | 4 ++-- services/agent/src/answer_evaluation.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 943344247..5ce32a570 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -140,7 +140,11 @@ The runner treats the generated answer as one untrusted JSON string value and te ignore directives inside it (a prompt boundary, not proof of injection immunity). The judge uses the selected citation list and fragments to assess support; inline IDs and verbatim quotes are not required, but selecting citations alone does not establish support. It classifies actual answer -behavior rather than copying the expected label, and treats an appropriate refusal, evidence +behavior rather than copying the expected label. Overlapping behaviors use the order +`refuse > clarify > no_evidence > cite`: a cited refusal remains `refuse`, and citations +do not override rejection, a missing-input question or reported evidence insufficiency. +The runner retains the judge's returned label and records mismatches; it never rewrites +labels to match expectations. It treats an appropriate refusal, evidence limitation, or clarification as a completed response when it addresses the question. Synthetic fragments do not substantiate claims about real submissions. Both passes receive each fragment as a single JSON object using the existing source projection, including trusted snapshot diff --git a/services/agent/data/repository_corpus_manifest.json b/services/agent/data/repository_corpus_manifest.json index 21b5c5101..9fd0b058b 100644 --- a/services/agent/data/repository_corpus_manifest.json +++ b/services/agent/data/repository_corpus_manifest.json @@ -29,8 +29,8 @@ }, { "doc_id": "repository-development", - "version": "sha256-1c4c80e280e0f713eaf6d3db0a951c6b44b7d40df88d9235c71d9bb945eba0ce", - "chunk_id": "repository-development:sha256-1c4c80e280e0f713eaf6d3db0a951c6b44b7d40df88d9235c71d9bb945eba0ce:1", + "version": "sha256-f7353248075e8083d8402ef30692795a6ce0ecadfef2f14861c7186584191a1e", + "chunk_id": "repository-development:sha256-f7353248075e8083d8402ef30692795a6ce0ecadfef2f14861c7186584191a1e:1", "source_path": "docs/DEVELOPMENT.md", "access_scope": "repository-public", "sample_kind": "real", diff --git a/services/agent/src/answer_evaluation.py b/services/agent/src/answer_evaluation.py index c9986d2d6..2affbb148 100644 --- a/services/agent/src/answer_evaluation.py +++ b/services/agent/src/answer_evaluation.py @@ -95,10 +95,15 @@ "material claims; a nonempty citation list alone does not establish support, " "and a supported paraphrase need not be a verbatim quote. Synthetic examples " "support explanations about those examples, not claims about a real submission. " - "cite means the answer uses its cited fragments to answer the question; " - "no_evidence means the answer reports insufficient relevant evidence; " - "refuse means it rejects the request; clarify means it asks for missing input. " - "A cited explanation of evidence limits can still be cite. Judge " + "Apply exactly one label, with precedence refuse > clarify > no_evidence > cite. " + "refuse means the answer rejects the requested action, even with cited evidence limits; " + "clarify means it asks for missing input without rejecting the request; " + "no_evidence means the answer reports insufficient relevant evidence without " + "rejecting the request or asking for missing input; cite means it uses cited fragments " + "to answer the question and none of the preceding behaviors applies. " + "A citation never overrides a higher-priority behavior. For example, " + "'The cited status fragment cannot identify a code line, so I cannot name one.' " + "is refuse, even when its citation is supported. Judge " "answer_completed by whether the question is addressed, including an appropriate " "refusal, evidence limitation, or clarification, not by whether unavailable " "private details were supplied. " From 9aa2d691752ac422152ba595fb7d7ba0ab7ddd7d Mon Sep 17 00:00:00 2001 From: DavidHLP Date: Sat, 10 Oct 2026 20:29:03 -0700 Subject: [PATCH 3/3] test(agent): preserve mismatch assertion in its original regression --- services/agent/tests/test_answer_evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/agent/tests/test_answer_evaluation.py b/services/agent/tests/test_answer_evaluation.py index 78f89da58..e4039bd85 100644 --- a/services/agent/tests/test_answer_evaluation.py +++ b/services/agent/tests/test_answer_evaluation.py @@ -316,6 +316,7 @@ def test_a_behavior_mismatch_is_recorded_not_hidden() -> None: assert row.observed_behavior == "cite" assert row.behavior_match is False assert row.citation_support == "unsupported" + assert row.answer_completion == "incomplete" def test_a_cited_refusal_keeps_refusal_behavior() -> None: @@ -337,7 +338,6 @@ def test_a_cited_refusal_keeps_refusal_behavior() -> None: assert "refuse > clarify > no_evidence > cite" in prompt assert "A citation never overrides a higher-priority behavior" in prompt assert json.loads(prompt.rsplit("\nANSWER_JSON ", 1)[1]) == text - assert row.answer_completion == "incomplete" def test_an_unknown_behavior_label_is_a_protocol_failure() -> None: