diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index ef972c1bdb..5a7f5f3234 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -406,12 +406,86 @@ arguments, observations, sensitive values, input paths, task text, verifier outp or trajectory content. Invalid private input returns a generic fail-closed error so JSON parser details cannot echo private data. +Some benchmarks expose a case-scoped submission API to the solver through task-owned +environment variables. When that access is part of the task contract, declare the +exact namespace and variable names in the integrity policy instead of waiving the +credential scanner: + +```json +{ + "schema_version": "benchmark_integrity_policy_v0", + "policy_id": "fixture-task-submission", + "task_environment_access": { + "schema_version": "benchmark_task_environment_access_v0", + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "", + "authority": "benchmark_task", + "purpose": "task_submission", + "namespace": "TASKCAP_", + "variable_names": ["TASKCAP_JUDGE_URL", "TASKCAP_TOKEN"] + } +} +``` + +This contract authorizes only the runner-owned +`benchmark_task_environment_read` typed operation. Its request and result must bind +the authorization id and digest and carry the same non-empty subset of declared +variable names. Shell `env`, `printenv`, and filtered enumeration remain credential +probes: even an absolute executable path can be replaced by an exported shell +function. The runner must still attest `provider_credential_shell_excluded=true`; an +undeclared name, mismatched typed result, provider credential value, or missing +isolation attestation remains disqualifying. The receipt publishes only the +authorization id and digest, purpose, counts and a digest of sorted variable names. +Raw names and values remain in the private trajectory. The declaration must be +derived from the immutable benchmark task contract; the toolkit validates its bounded +shape but does not grant that task authority. Because this is a qualification-time +task permission, it is excluded from the digest of the run-admitted evaluation policy. +The private trajectory records this exact request/result shape: + +```json +{ + "function_name": "benchmark_task_environment_read", + "arguments": { + "schema_version": "benchmark_task_environment_read_v0", + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "", + "variable_names": ["TASKCAP_TOKEN"] + }, + "observation": { + "schema_version": "benchmark_task_environment_read_result_v0", + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "", + "values": {"TASKCAP_TOKEN": ""} + } +} +``` + +The default protocol remains blind: the verifier starts after the agent and official +feedback is hidden. A benchmark that intentionally returns periodic evaluation during +solving must preregister an `evaluation_protocol` in the integrity policy with +`feedback=declared`, `timing=during_solving`, and exact channels plus output kinds. +For that policy only, qualification replaces the two blind-protocol attestations with +`evaluation_protocol_enforced`, `evaluation_log_complete`, and +`verifier_started_after_submission`. The private runner evidence must bind the exact +policy and trajectory digests, prove strictly ordered submissions and declared +deliveries, and show a complete acknowledged feedback transport. At most the final +submission may finish evaluation after the solver ends, and it cannot deliver feedback +back into the solving phase. If that is the only submission, the complete evidence has +an explicit empty `deliveries` list and integer `intent_count=ack_count=0`; submission +request/response acknowledgements and their record digests are still required. Empty +deliveries never excuse missing feedback for evaluations completed during solving. +The public evidence digest binds both the protocol log and +the feedback transport; the receipt otherwise contains only digests and +submission/delivery counts. A prose claim that feedback was expected, an unbound score +log, an undeclared channel, or an incomplete transport does not relax the blind default. + Qualification rejects a run when it detects any of the following: - post-run agent confirmation that restricted answer, out-of-scope task-source, hidden-test, verifier, other-trial, or controller-private material was both disclosed and causally used during solving or validation; -- host escape, credential probing or exposure, or shell network access; +- host escape, credential probing outside an exact task-environment contract, + credential exposure, or shell network access; - malformed or incomplete ATIF tool evidence; - missing runner authority or any required runtime isolation attestation. diff --git a/loopx/capabilities/benchmark_toolkit/__init__.py b/loopx/capabilities/benchmark_toolkit/__init__.py index bbe411079a..1ad24e4e86 100644 --- a/loopx/capabilities/benchmark_toolkit/__init__.py +++ b/loopx/capabilities/benchmark_toolkit/__init__.py @@ -60,6 +60,11 @@ render_benchmark_experiment_board_markdown, upsert_benchmark_experiment_board_row, ) +from .environment_access import ( + BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_TOOL, +) from .continuation import ( BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION, BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION, @@ -86,6 +91,8 @@ BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION, BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION, BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, INTEGRITY_EVIDENCE_CATEGORIES, REQUIRED_RUNTIME_ATTESTATIONS, RestrictedAccessAdjudicationDecision, @@ -242,6 +249,11 @@ "BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION", "BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION", "BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION", + "BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION", + "BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION", + "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION", + "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION", + "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL", "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION", "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION", "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION", diff --git a/loopx/capabilities/benchmark_toolkit/catalog_entry.py b/loopx/capabilities/benchmark_toolkit/catalog_entry.py index be275eecb0..4381c441c9 100644 --- a/loopx/capabilities/benchmark_toolkit/catalog_entry.py +++ b/loopx/capabilities/benchmark_toolkit/catalog_entry.py @@ -825,10 +825,58 @@ "qualified_with_warning." ), }, + "task_environment_access": { + "policy_schema": "benchmark_task_environment_access_v0", + "receipt_schema": "benchmark_task_environment_access_receipt_v0", + "authority": "benchmark_task", + "purpose": "task_submission", + "hint": ( + "Declare an exact task-owned namespace and variable-name set only " + "when the benchmark task contract exposes a solver submission " + "capability. Access must use the runner-owned typed read operation, " + "and the runner must separately attest that provider credentials " + "are excluded from the solver shell." + ), + "fail_closed_on": [ + "broad_environment_enumeration", + "undeclared_variable_name", + "dynamic_namespace_filter", + "provider_credential_value", + "missing_provider_credential_isolation_attestation", + ], + "public_receipt_boundary": ( + "authorization id and digest, purpose, counts and a variable-name " + "digest only; raw names and values stay private" + ), + }, + "declared_feedback_protocol": { + "default": "official_feedback_blinded", + "opt_in": { + "feedback": "declared", + "timing": "during_solving", + }, + "required_runner_attestations": [ + "evaluation_protocol_enforced", + "evaluation_log_complete", + "verifier_started_after_submission", + ], + "evidence_binding": [ + "exact_policy_sha256", + "exact_trajectory_sha256", + "ordered_submissions", + "declared_channel_deliveries", + "acknowledged_feedback_transport", + ], + "boundary": ( + "Only a preregistered policy plus complete private runner evidence " + "replaces the blind-protocol attestations; prose or partial score " + "logs remain unqualified." + ), + }, "role_boundary": { "solver": ( "must not access hidden tests, grader or verifier sources, expected " - "answers, or official feedback during the solving phase" + "answers, or undeclared official feedback during the solving phase" ), "active_campaign_monitor": ( "may inspect the solver-owned trajectory and exact-job runtime while " @@ -969,6 +1017,26 @@ "module": "loopx.capabilities.benchmark_toolkit.integrity", "doc": "loopx/capabilities/benchmark_toolkit/README.md", }, + { + "schema_version": "benchmark_task_environment_access_v0", + "module": "loopx.capabilities.benchmark_toolkit.integrity", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, + { + "schema_version": "benchmark_task_environment_access_receipt_v0", + "module": "loopx.capabilities.benchmark_toolkit.integrity", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, + { + "schema_version": "benchmark_task_environment_read_v0", + "module": "loopx.capabilities.benchmark_toolkit.environment_access", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, + { + "schema_version": "benchmark_task_environment_read_result_v0", + "module": "loopx.capabilities.benchmark_toolkit.environment_access", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, { "schema_version": "benchmark_restricted_access_adjudication_v0", "module": "loopx.capabilities.benchmark_toolkit.integrity", diff --git a/loopx/capabilities/benchmark_toolkit/environment_access.py b/loopx/capabilities/benchmark_toolkit/environment_access.py index 16e3c2858a..9612082134 100644 --- a/loopx/capabilities/benchmark_toolkit/environment_access.py +++ b/loopx/capabilities/benchmark_toolkit/environment_access.py @@ -22,6 +22,13 @@ _SHELL_EXECUTABLES = frozenset({"bash", "dash", "fish", "ksh", "sh", "zsh"}) _PYTHON_EXECUTABLE_PATTERN = re.compile(r"(?i)^(?:python(?:[0-9.]+)?|pypy[0-9.]*)$") _JAVASCRIPT_EXECUTABLES = frozenset({"node", "nodejs"}) +BENCHMARK_TASK_ENVIRONMENT_READ_TOOL = "benchmark_task_environment_read" +BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION = ( + "benchmark_task_environment_read_v0" +) +BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION = ( + "benchmark_task_environment_read_result_v0" +) _JavascriptToken = tuple[str, str] @@ -455,12 +462,109 @@ def _javascript_environment_probe(code: str) -> bool: return False -def _shell_segment_has_credential_probe(tokens: tuple[str, ...]) -> bool: +def _shell_wrapper_command( + executable: str, + arguments: tuple[str, ...], +) -> tuple[str, ...]: + remaining = list(arguments) + if executable == "command": + while remaining and remaining[0].startswith("-"): + option = remaining.pop(0) + if option in {"-v", "-V"}: + return () + if option == "--": + break + return tuple(remaining) + if executable == "exec": + while remaining and remaining[0].startswith("-"): + option = remaining.pop(0) + if option == "--": + break + if option in {"-a", "--argv0"}: + if not remaining: + return () + remaining.pop(0) + return tuple(remaining) + return () + + +def _xargs_command(arguments: tuple[str, ...]) -> tuple[str, ...]: + """Return the utility position from a bounded GNU/POSIX xargs argv.""" + + long_options_with_value = { + "--arg-file", + "--delimiter", + "--max-args", + "--max-procs", + "--max-chars", + "--process-slot-var", + } + long_options_with_optional_attached_value = { + "--eof", + "--replace", + "--max-lines", + } + short_options_with_value = frozenset("adEILnPs") + short_options_with_optional_attached_value = frozenset("eil") + short_options_without_value = frozenset("0ortpx") + index = 0 + while index < len(arguments): + argument = arguments[index] + if argument == "--": + return arguments[index + 1 :] + if not argument.startswith("-") or argument == "-": + return arguments[index:] + if argument.startswith("--"): + option, separator, _ = argument.partition("=") + if option in {"--help", "--version"}: + return () + if option in long_options_with_value and not separator: + index += 2 + elif option in long_options_with_optional_attached_value: + index += 1 + else: + index += 1 + continue + + cluster = argument[1:] + consume_next = False + for offset, option in enumerate(cluster): + if option in short_options_without_value: + continue + if option in short_options_with_value: + consume_next = offset == len(cluster) - 1 + break + if option in short_options_with_optional_attached_value: + break + # Unknown xargs options fail closed by treating the next token as a + # possible utility rather than skipping an unproven value. + break + index += 2 if consume_next else 1 + return () + + +def _shell_segment_has_credential_probe( + tokens: tuple[str, ...], + *, + depth: int = 0, +) -> bool: + if depth >= 8: + return True executable, arguments = _shell_executable(tokens) if not executable: return False if executable in {"env", "printenv"}: return True + if executable in {"command", "exec"}: + wrapped = _shell_wrapper_command(executable, arguments) + return bool(wrapped) and _shell_segment_has_credential_probe( + wrapped, depth=depth + 1 + ) + if executable == "xargs": + wrapped = _xargs_command(arguments) + return bool(wrapped) and _shell_segment_has_credential_probe( + wrapped, depth=depth + 1 + ) if any(_PROC_ENVIRON_PATTERN.search(argument) for argument in arguments): return True if executable in _SHELL_EXECUTABLES: @@ -490,6 +594,11 @@ def _shell_command_has_credential_probe(command: str) -> bool: def credential_probe_present(function_name: str, raw_arguments: object) -> bool: """Classify executed command fields without scanning source-search prose.""" + if ( + function_name.strip().casefold() + == BENCHMARK_TASK_ENVIRONMENT_READ_TOOL.casefold() + ): + return True commands = _command_strings(function_name, raw_arguments) if any(_shell_command_has_credential_probe(command) for command in commands): return True @@ -500,3 +609,70 @@ def credential_probe_present(function_name: str, raw_arguments: object) -> bool: ): return _javascript_environment_probe(raw_arguments["input"]) return False + + +def authorized_task_environment_probe_count( + function_name: str, + raw_arguments: object, + raw_observation: object, + *, + authorization_id: str, + authorization_sha256: str, + variable_names: frozenset[str], +) -> int: + """Count one typed runner read covered by the task-owned env contract. + + Shell commands are intentionally never authorized here. Even an absolute + executable name can be shadowed by an exported shell function, so only a + runner-owned typed operation can prove which task values were requested and + returned. Raw names and values remain private in the trajectory. + """ + + if ( + function_name != BENCHMARK_TASK_ENVIRONMENT_READ_TOOL + or not isinstance(raw_arguments, Mapping) + or set(raw_arguments) + != { + "schema_version", + "authorization_id", + "authorization_sha256", + "variable_names", + } + or raw_arguments.get("schema_version") + != BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION + or raw_arguments.get("authorization_id") != authorization_id + or raw_arguments.get("authorization_sha256") != authorization_sha256 + ): + return 0 + requested_names = raw_arguments.get("variable_names") + if ( + not isinstance(requested_names, list) + or not requested_names + or not all(isinstance(name, str) for name in requested_names) + or len(requested_names) != len(set(requested_names)) + or not set(requested_names).issubset(variable_names) + ): + return 0 + if ( + not isinstance(raw_observation, Mapping) + or set(raw_observation) + != { + "schema_version", + "authorization_id", + "authorization_sha256", + "values", + } + or raw_observation.get("schema_version") + != BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION + or raw_observation.get("authorization_id") != authorization_id + or raw_observation.get("authorization_sha256") != authorization_sha256 + ): + return 0 + values = raw_observation.get("values") + if ( + not isinstance(values, Mapping) + or set(values) != set(requested_names) + or not all(isinstance(value, str) for value in values.values()) + ): + return 0 + return 1 diff --git a/loopx/capabilities/benchmark_toolkit/integrity.py b/loopx/capabilities/benchmark_toolkit/integrity.py index 6b5990e5c4..05f0b2c740 100644 --- a/loopx/capabilities/benchmark_toolkit/integrity.py +++ b/loopx/capabilities/benchmark_toolkit/integrity.py @@ -10,10 +10,14 @@ from collections import Counter from collections.abc import Iterable, Mapping from enum import Enum -from typing import Any +from typing import Any, TypeGuard, cast from urllib.parse import urlsplit -from .environment_access import _heredoc_delimiters, credential_probe_present +from .environment_access import ( + _heredoc_delimiters, + authorized_task_environment_probe_count, + credential_probe_present, +) BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION = "benchmark_integrity_policy_v0" BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION = ( @@ -22,6 +26,12 @@ BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION = ( "benchmark_integrity_qualification_v0" ) +BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION = ( + "benchmark_task_environment_access_v0" +) +BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION = ( + "benchmark_task_environment_access_receipt_v0" +) BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION = ( "benchmark_restricted_access_adjudication_v0" ) @@ -53,6 +63,11 @@ "verifier_started_after_agent", "official_feedback_blinded", ) +_DECLARED_FEEDBACK_RUNTIME_ATTESTATIONS = ( + "evaluation_protocol_enforced", + "evaluation_log_complete", + "verifier_started_after_submission", +) NETWORK_ACCESS_MODES = ("denied", "loopback_only", "permitted_solving") DEFAULT_NETWORK_ACCESS = "denied" @@ -71,9 +86,15 @@ "case_local_control_state", "canonical_control_state_root", "independent_verifier", +) +_BLINDED_EVALUATION_RUNTIME_ATTESTATIONS = ( "verifier_started_after_agent", "official_feedback_blinded", ) +_EVALUATION_PROTOCOL_OUTPUT_KINDS = frozenset( + {"score", "validation", "metrics", "summary", "diagnostics"} +) +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") _DEFAULT_DENIED_ARGUMENT_MARKERS: dict[str, tuple[str, ...]] = { "restricted_answer_source_request": ( @@ -135,6 +156,8 @@ # Unknown tools remain fail-closed and continue through the access-request scan. _NON_ACCESS_CONTROL_TOOLS = frozenset({"update_plan"}) _PUBLIC_EVIDENCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") +_TASK_ENVIRONMENT_NAMESPACE_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,30}_$") +_TASK_ENVIRONMENT_VARIABLE_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,79}$") _REQUIRED_ADJUDICATION_REVIEW_SURFACES = frozenset( {"solver_trajectory", "tool_results", "final_workspace"} ) @@ -208,7 +231,11 @@ def _marker_present(text: str, marker: str) -> bool: ) -def required_runtime_attestations(network_access: str) -> tuple[str, ...]: +def required_runtime_attestations( + network_access: str, + *, + declared_feedback: bool = False, +) -> tuple[str, ...]: """Return the runner attestations required for one network access mode. ``denied`` (default) requires ``shell_network_denied`` for offline coding @@ -217,7 +244,9 @@ def required_runtime_attestations(network_access: str) -> tuple[str, ...]: ``permitted_solving`` is for web-research benchmarks: the shell may use the network during the solving phase, but the runner must attest ``network_permitted_solving`` instead and every restricted-resource denial - still applies. + still applies. The default evaluation contract remains blinded. A validated + declared-feedback policy replaces only its two evaluation-order attestations + with the protocol, log, and post-submission verifier attestations. """ mode = ( @@ -225,22 +254,48 @@ def required_runtime_attestations(network_access: str) -> tuple[str, ...]: if network_access in NETWORK_ACCESS_MODES else DEFAULT_NETWORK_ACCESS ) + evaluation_attestations = ( + _DECLARED_FEEDBACK_RUNTIME_ATTESTATIONS + if declared_feedback + else _BLINDED_EVALUATION_RUNTIME_ATTESTATIONS + ) if mode == "permitted_solving": - return (*_COMMON_RUNTIME_ATTESTATIONS, NETWORK_PERMITTED_SOLVING_ATTESTATION) + return ( + *_COMMON_RUNTIME_ATTESTATIONS, + *evaluation_attestations, + NETWORK_PERMITTED_SOLVING_ATTESTATION, + ) if mode == "loopback_only": - return (*_COMMON_RUNTIME_ATTESTATIONS, LOOPBACK_ONLY_ATTESTATION) - return (*_COMMON_RUNTIME_ATTESTATIONS, "shell_network_denied") + return ( + *_COMMON_RUNTIME_ATTESTATIONS, + *evaluation_attestations, + LOOPBACK_ONLY_ATTESTATION, + ) + return ( + *_COMMON_RUNTIME_ATTESTATIONS, + *evaluation_attestations, + "shell_network_denied", + ) def _validated_policy( policy: Mapping[str, Any] | None, -) -> tuple[str, bool, dict[str, tuple[str, ...]], str]: +) -> tuple[ + str, + bool, + dict[str, tuple[str, ...]], + str, + dict[str, object] | None, + dict[str, object] | None, +]: if policy is None: return ( "default", False, dict(_DEFAULT_DENIED_ARGUMENT_MARKERS), DEFAULT_NETWORK_ACCESS, + None, + None, ) if policy.get("schema_version") != BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION: raise ValueError("benchmark_integrity_policy_schema_mismatch") @@ -274,7 +329,152 @@ def _validated_policy( normalized.append(text) if normalized: markers[category] = (*markers.get(category, ()), *normalized) - return policy_id, policy_id_path_like, markers, raw_network_access + task_environment_access = _validated_task_environment_access( + policy.get("task_environment_access") + ) + runtime_policy_projection = { + key: value + for key, value in policy.items() + if key != "task_environment_access" + } + evaluation_protocol = _validated_evaluation_protocol( + policy.get("evaluation_protocol"), + policy_sha256=_sha256_text(_canonical_text(runtime_policy_projection)), + ) + return ( + policy_id, + policy_id_path_like, + markers, + raw_network_access, + task_environment_access, + evaluation_protocol, + ) + + +def _validated_task_environment_access( + raw_access: object, +) -> dict[str, object] | None: + if raw_access is None: + return None + if not isinstance(raw_access, Mapping): + raise TypeError("benchmark_task_environment_access_invalid") + if ( + raw_access.get("schema_version") + != BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION + ): + raise ValueError("benchmark_task_environment_access_schema_mismatch") + authorization_id = str(raw_access.get("authorization_id") or "").strip() + if _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(authorization_id) is None: + raise ValueError("benchmark_task_environment_access_authorization_id_invalid") + authorization_sha256 = str(raw_access.get("authorization_sha256") or "") + if _SHA256_PATTERN.fullmatch(authorization_sha256) is None: + raise ValueError( + "benchmark_task_environment_access_authorization_sha256_invalid" + ) + if raw_access.get("authority") != "benchmark_task": + raise ValueError("benchmark_task_environment_access_authority_invalid") + if raw_access.get("purpose") != "task_submission": + raise ValueError("benchmark_task_environment_access_purpose_invalid") + namespace = str(raw_access.get("namespace") or "").strip() + if _TASK_ENVIRONMENT_NAMESPACE_PATTERN.fullmatch(namespace) is None: + raise ValueError("benchmark_task_environment_access_namespace_invalid") + raw_names = raw_access.get("variable_names") + if ( + not isinstance(raw_names, list) + or not raw_names + or len(raw_names) > 32 + or not all(isinstance(name, str) for name in raw_names) + ): + raise ValueError("benchmark_task_environment_access_variables_invalid") + variable_names = tuple(sorted(dict.fromkeys(raw_names))) + if len(variable_names) != len(raw_names) or any( + _TASK_ENVIRONMENT_VARIABLE_PATTERN.fullmatch(name) is None + for name in variable_names + ): + raise ValueError("benchmark_task_environment_access_variables_invalid") + if any(not name.startswith(namespace) for name in variable_names): + raise ValueError( + "benchmark_task_environment_access_variable_outside_namespace" + ) + return { + "authorization_id": authorization_id, + "authorization_sha256": authorization_sha256, + "authority": "benchmark_task", + "purpose": "task_submission", + "namespace": namespace, + "variable_names": variable_names, + } + + +def _validated_evaluation_protocol( + raw_protocol: object, + *, + policy_sha256: str, +) -> dict[str, object] | None: + if raw_protocol is None: + return None + if not isinstance(raw_protocol, Mapping): + raise TypeError("benchmark_evaluation_protocol_invalid") + protocol_id = str(raw_protocol.get("protocol_id") or "").strip() + if _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(protocol_id) is None: + raise ValueError("benchmark_evaluation_protocol_id_invalid") + if raw_protocol.get("feedback") != "declared": + raise ValueError("benchmark_evaluation_protocol_feedback_invalid") + if raw_protocol.get("timing") != "during_solving": + raise ValueError("benchmark_evaluation_protocol_timing_invalid") + raw_channels = raw_protocol.get("channels") + if ( + not isinstance(raw_channels, list) + or not raw_channels + or len(raw_channels) > 8 + ): + raise ValueError("benchmark_evaluation_protocol_channels_invalid") + channels: list[dict[str, object]] = [] + seen_channel_ids: set[str] = set() + for raw_channel in raw_channels: + if not isinstance(raw_channel, Mapping): + raise TypeError("benchmark_evaluation_protocol_channel_invalid") + channel_id = str(raw_channel.get("channel_id") or "").strip() + if ( + _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(channel_id) is None + or channel_id in seen_channel_ids + ): + raise ValueError("benchmark_evaluation_protocol_channel_id_invalid") + seen_channel_ids.add(channel_id) + if raw_channel.get("phase") != "solving": + raise ValueError("benchmark_evaluation_protocol_channel_phase_invalid") + raw_output_kinds = raw_channel.get("output_kinds") + if ( + not isinstance(raw_output_kinds, list) + or not raw_output_kinds + or len(raw_output_kinds) > len(_EVALUATION_PROTOCOL_OUTPUT_KINDS) + or not all(isinstance(kind, str) for kind in raw_output_kinds) + ): + raise ValueError( + "benchmark_evaluation_protocol_output_kinds_invalid" + ) + output_kinds = tuple(sorted(dict.fromkeys(raw_output_kinds))) + if ( + len(output_kinds) != len(raw_output_kinds) + or not set(output_kinds).issubset(_EVALUATION_PROTOCOL_OUTPUT_KINDS) + ): + raise ValueError( + "benchmark_evaluation_protocol_output_kinds_invalid" + ) + channels.append( + { + "channel_id": channel_id, + "phase": "solving", + "output_kinds": output_kinds, + } + ) + return { + "protocol_id": protocol_id, + "feedback": "declared", + "timing": "during_solving", + "channels": tuple(channels), + "policy_sha256": policy_sha256, + } def _validated_restricted_access_adjudication( @@ -617,6 +817,262 @@ def _network_request_scope(arguments: object) -> NetworkRequestScope: return scope +def _nonnegative_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _positive_int(value: object) -> TypeGuard[int]: + return _nonnegative_int(value) and value > 0 + + +def _validated_declared_feedback_runtime( + *, + trajectory: Mapping[str, Any], + runtime_attestation: Mapping[str, Any], + protocol: Mapping[str, object], +) -> tuple[dict[str, object] | None, list[str]]: + """Validate the private proof for task-declared feedback during solving.""" + + failures: list[str] = [] + raw_runtime = runtime_attestation.get("evaluation_protocol") + if not isinstance(raw_runtime, Mapping): + return None, ["evaluation_protocol_runtime_evidence_missing"] + policy_sha256 = raw_runtime.get("policy_sha256") + if policy_sha256 != protocol["policy_sha256"]: + failures.append("evaluation_protocol_policy_mismatch") + if raw_runtime.get("trajectory_sha256") != _sha256_text( + _canonical_text(trajectory) + ): + failures.append("evaluation_protocol_trajectory_mismatch") + run_id = str(raw_runtime.get("run_id") or "") + if _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(run_id) is None: + failures.append("evaluation_protocol_run_id_invalid") + + admitted_at = raw_runtime.get("policy_admitted_at_ms") + solver_started_at = raw_runtime.get("solver_started_at_ms") + solver_ended_at = raw_runtime.get("solver_ended_at_ms") + if ( + _positive_int(admitted_at) + and _positive_int(solver_started_at) + and _positive_int(solver_ended_at) + and admitted_at <= solver_started_at <= solver_ended_at + ): + timeline: tuple[int, int, int] | None = ( + admitted_at, + solver_started_at, + solver_ended_at, + ) + else: + timeline = None + failures.append("evaluation_protocol_timeline_invalid") + solver_start_ms = timeline[1] if timeline is not None else 0 + solver_end_ms = timeline[2] if timeline is not None else 0 + + raw_submissions_value = raw_runtime.get("submissions") + submissions = ( + raw_submissions_value + if isinstance(raw_submissions_value, list) + else [] + ) + submissions_by_id: dict[str, dict[str, int]] = {} + previous_captured_at: int | None = None + post_solver_completion_indexes: list[int] = [] + submissions_valid = ( + isinstance(raw_submissions_value, list) + and bool(submissions) + and len(submissions) <= 10_000 + ) + if submissions_valid: + for submission_index, raw_submission in enumerate(submissions): + if not isinstance(raw_submission, Mapping): + submissions_valid = False + break + submission_id = str(raw_submission.get("submission_id") or "") + captured_at = raw_submission.get("captured_at_ms") + evaluation_started_at = raw_submission.get("evaluation_started_at_ms") + evaluation_completed_at = raw_submission.get( + "evaluation_completed_at_ms" + ) + if ( + _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(submission_id) is None + or submission_id in submissions_by_id + or _SHA256_PATTERN.fullmatch( + str(raw_submission.get("submission_sha256") or "") + ) + is None + or not _positive_int(captured_at) + or not _positive_int(evaluation_started_at) + or not _positive_int(evaluation_completed_at) + or timeline is None + or not ( + solver_start_ms + <= captured_at + <= solver_end_ms + and captured_at + <= evaluation_started_at + <= evaluation_completed_at + ) + or ( + previous_captured_at is not None + and captured_at <= previous_captured_at + ) + ): + submissions_valid = False + break + submissions_by_id[submission_id] = { + "evaluation_completed_at_ms": evaluation_completed_at + } + previous_captured_at = captured_at + if evaluation_completed_at > solver_end_ms: + post_solver_completion_indexes.append(submission_index) + if submissions_valid and ( + len(post_solver_completion_indexes) > 1 + or ( + post_solver_completion_indexes + and post_solver_completion_indexes[0] != len(submissions) - 1 + ) + ): + submissions_valid = False + if not submissions_valid: + failures.append("evaluation_protocol_submissions_invalid") + + protocol_channels = cast( + tuple[Mapping[str, object], ...], protocol["channels"] + ) + declared_channels: dict[str, frozenset[str]] = { + str(channel["channel_id"]): frozenset( + cast(tuple[str, ...], channel["output_kinds"]) + ) + for channel in protocol_channels + } + raw_deliveries_value = raw_runtime.get("deliveries") + deliveries = ( + raw_deliveries_value + if isinstance(raw_deliveries_value, list) + else [] + ) + deliveries_valid = ( + submissions_valid + and isinstance(raw_deliveries_value, list) + and len(deliveries) <= 50_000 + ) + seen_deliveries: set[tuple[str, str, str]] = set() + delivered_submission_ids: set[str] = set() + if deliveries_valid: + for raw_delivery in deliveries: + if not isinstance(raw_delivery, Mapping): + deliveries_valid = False + break + submission_id = str(raw_delivery.get("submission_id") or "") + channel_id = str(raw_delivery.get("channel_id") or "") + feedback_sha256 = str(raw_delivery.get("feedback_sha256") or "") + delivered_at = raw_delivery.get("delivered_at_ms") + raw_output_kinds = raw_delivery.get("output_kinds") + output_kind_values = ( + tuple(raw_output_kinds) + if isinstance(raw_output_kinds, list) + and raw_output_kinds + and all(isinstance(kind, str) for kind in raw_output_kinds) + else () + ) + output_kinds = frozenset(output_kind_values) + identity = (submission_id, channel_id, feedback_sha256) + submission = submissions_by_id.get(submission_id) + if ( + submission is None + or channel_id not in declared_channels + or raw_delivery.get("phase") != "solving" + or not output_kinds + or len(output_kinds) != len(output_kind_values) + or not output_kinds.issubset(declared_channels[channel_id]) + or _SHA256_PATTERN.fullmatch(feedback_sha256) is None + or identity in seen_deliveries + or not _positive_int(delivered_at) + or timeline is None + or not ( + submission["evaluation_completed_at_ms"] + <= delivered_at + <= solver_end_ms + ) + ): + deliveries_valid = False + break + seen_deliveries.add(identity) + delivered_submission_ids.add(submission_id) + required_delivery_submission_ids = { + submission_id + for submission_id, submission in submissions_by_id.items() + if submission["evaluation_completed_at_ms"] <= solver_end_ms + } + # An explicit empty list is complete evidence only when the final + # submission drains after solving, so no in-solving delivery is due. + if delivered_submission_ids != required_delivery_submission_ids: + deliveries_valid = False + if not deliveries_valid: + failures.append("evaluation_protocol_deliveries_invalid") + + raw_transport = runtime_attestation.get("feedback_transport") + transport = raw_transport if isinstance(raw_transport, Mapping) else None + transport_valid = deliveries_valid and transport is not None + if transport_valid and transport is not None: + request_count = transport.get("request_count") + response_ack_count = transport.get("response_ack_count") + intent_count = transport.get("intent_count") + ack_count = transport.get("ack_count") + record_digests = transport.get("record_digests") + if ( + not _positive_int(request_count) + or not _positive_int(response_ack_count) + or not _nonnegative_int(intent_count) + or not _nonnegative_int(ack_count) + ): + transport_valid = False + else: + transport_valid = bool( + request_count == response_ack_count + and request_count >= len(submissions_by_id) + and intent_count == len(deliveries) + and ack_count == intent_count + and str(transport.get("run_id") or "") == run_id + and transport.get("deliveries_sha256") + == _sha256_text(_canonical_text(deliveries)) + and _SHA256_PATTERN.fullmatch( + str(transport.get("source_manifest_sha256") or "") + ) + is not None + and isinstance(record_digests, Mapping) + and len(record_digests) == request_count + response_ack_count + and all( + isinstance(key, str) + and _SHA256_PATTERN.fullmatch(str(value or "")) is not None + for key, value in record_digests.items() + ) + ) + if not transport_valid: + failures.append("evaluation_protocol_transport_invalid") + + if failures: + return None, failures + return ( + { + "feedback": "declared", + "timing": "during_solving", + "policy_sha256": policy_sha256, + "submission_count": len(submissions), + "delivery_count": len(deliveries), + "evidence_sha256": _sha256_text( + _canonical_text( + { + "evaluation_protocol": raw_runtime, + "feedback_transport": transport, + } + ) + ), + }, + [], + ) + + def build_benchmark_integrity_qualification( *, trajectory: Mapping[str, Any], @@ -632,7 +1088,14 @@ def build_benchmark_integrity_qualification( absence of a suspicious tool call cannot prove that isolation existed. """ - policy_id, policy_id_path_like, markers, network_access = _validated_policy(policy) + ( + policy_id, + policy_id_path_like, + markers, + network_access, + task_environment_access, + evaluation_protocol, + ) = _validated_policy(policy) adjudication = _validated_restricted_access_adjudication( restricted_access_adjudication ) @@ -673,6 +1136,7 @@ def build_benchmark_integrity_qualification( tool_call_count = 0 observation_count = 0 invalid_tool_call_count = 0 + authorized_task_environment_probe_count_total = 0 for index, raw_step in enumerate(steps, start=1): if not isinstance(raw_step, Mapping): structural_failures.append("trajectory_step_invalid") @@ -687,8 +1151,12 @@ def build_benchmark_integrity_qualification( invalid_tool_call_count += 1 continue tool_call_count += 1 + raw_function_name = raw_call.get("function_name") function_name = _safe_label( - raw_call.get("function_name") or "unknown", limit=80 + raw_function_name or "unknown", limit=80 + ) + exact_function_name = ( + raw_function_name if isinstance(raw_function_name, str) else "" ) raw_arguments = raw_call.get("arguments") or {} arguments = _canonical_text(raw_arguments) @@ -708,8 +1176,42 @@ def build_benchmark_integrity_qualification( for text in argument_texts ) } - if credential_probe_present(function_name, raw_arguments): - categories.add("credential_probe") + if credential_probe_present(exact_function_name, raw_arguments): + authorized_probe_count = 0 + if ( + task_environment_access is not None + and runtime_attestation.get( + "provider_credential_shell_excluded" + ) + is True + ): + authorized_probe_count = ( + authorized_task_environment_probe_count( + exact_function_name, + raw_arguments, + raw_step.get("observation"), + authorization_id=str( + task_environment_access["authorization_id"] + ), + authorization_sha256=str( + task_environment_access[ + "authorization_sha256" + ] + ), + variable_names=frozenset( + cast( + tuple[str, ...], + task_environment_access["variable_names"], + ) + ), + ) + ) + if authorized_probe_count: + authorized_task_environment_probe_count_total += ( + authorized_probe_count + ) + else: + categories.add("credential_probe") network_scope = _network_request_scope(raw_arguments) if network_scope is NetworkRequestScope.LOOPBACK: categories.add("loopback_network_request") @@ -754,9 +1256,24 @@ def build_benchmark_integrity_qualification( attestation_failures.append("runtime_attestation_schema_mismatch") if runtime_attestation.get("authority") != "runner": attestation_failures.append("runtime_attestation_authority_not_runner") - for field in required_runtime_attestations(network_access): + runtime_attestations = required_runtime_attestations( + network_access, + declared_feedback=evaluation_protocol is not None, + ) + for field in runtime_attestations: if runtime_attestation.get(field) is not True: attestation_failures.append(f"runtime_attestation_{field}_missing") + compact_evaluation_protocol: dict[str, object] | None = None + if evaluation_protocol is not None: + ( + compact_evaluation_protocol, + evaluation_protocol_failures, + ) = _validated_declared_feedback_runtime( + trajectory=trajectory, + runtime_attestation=runtime_attestation, + protocol=evaluation_protocol, + ) + attestation_failures.extend(evaluation_protocol_failures) counts = { category: int(evidence_counts.get(category, 0)) @@ -784,7 +1301,10 @@ def build_benchmark_integrity_qualification( confirmed_cheating_failures = ( [ "restricted_access_confirmed_cheating", - *restricted_access_review["suspected_categories"], + *cast( + tuple[str, ...], + restricted_access_review["suspected_categories"], + ), ] if cheating_detected else [] @@ -813,7 +1333,7 @@ def build_benchmark_integrity_qualification( else: classification = "integrity_policy_violation" - return { + receipt = { "ok": True, "schema_version": BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION, "benchmark_id": benchmark_id, @@ -833,7 +1353,7 @@ def build_benchmark_integrity_qualification( "network_access": network_access, "runtime_attestation_checks": { field: runtime_attestation.get(field) is True - for field in required_runtime_attestations(network_access) + for field in runtime_attestations }, "audit_coverage": { "trajectory_schema_version": _safe_label(schema_version, limit=40), @@ -861,3 +1381,37 @@ def build_benchmark_integrity_qualification( "confirmed_cheating_requires_disclosure_and_causal_use": True, }, } + if task_environment_access is not None: + variable_names = cast( + tuple[str, ...], task_environment_access["variable_names"] + ) + receipt["task_environment_access"] = { + "schema_version": ( + BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION + ), + "state": ( + "rejected" + if counts["credential_probe"] + else "authorized" + if authorized_task_environment_probe_count_total + else "declared" + ), + "authorization_id": task_environment_access["authorization_id"], + "authorization_sha256": task_environment_access[ + "authorization_sha256" + ], + "authority": task_environment_access["authority"], + "purpose": task_environment_access["purpose"], + "authorized_probe_count": ( + authorized_task_environment_probe_count_total + ), + "variable_name_count": len(variable_names), + "variable_names_sha256": _sha256_text( + _canonical_text(variable_names) + ), + "raw_names_recorded": False, + "raw_values_recorded": False, + } + if compact_evaluation_protocol is not None: + receipt["evaluation_protocol"] = compact_evaluation_protocol + return receipt diff --git a/tests/capabilities/test_benchmark_toolkit.py b/tests/capabilities/test_benchmark_toolkit.py index 2a8101edb5..e0526372e6 100644 --- a/tests/capabilities/test_benchmark_toolkit.py +++ b/tests/capabilities/test_benchmark_toolkit.py @@ -1,10 +1,11 @@ from __future__ import annotations import copy +import hashlib import json import os import subprocess -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path import pytest @@ -14,6 +15,11 @@ BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION, BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION, BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_TOOL, REQUIRED_RUNTIME_ATTESTATIONS, DockerContainerBindingError, build_benchmark_integrity_qualification, @@ -251,6 +257,31 @@ def test_catalog_exposes_post_run_case_insight_monitor_contract() -> None: } assert "disclosed=true plus causal_use=true" in adjudication["decision_rule"] + task_environment = analysis["task_environment_access"] + assert task_environment["policy_schema"] == ( + BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION + ) + assert task_environment["receipt_schema"] == ( + BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION + ) + assert task_environment["authority"] == "benchmark_task" + assert "provider credentials are excluded" in task_environment["hint"] + assert "broad_environment_enumeration" in task_environment["fail_closed_on"] + assert "raw names and values stay private" in ( + task_environment["public_receipt_boundary"] + ) + + declared_feedback = analysis["declared_feedback_protocol"] + assert declared_feedback["default"] == "official_feedback_blinded" + assert declared_feedback["opt_in"] == { + "feedback": "declared", + "timing": "during_solving", + } + assert "exact_policy_sha256" in declared_feedback["evidence_binding"] + assert "acknowledged_feedback_transport" in ( + declared_feedback["evidence_binding"] + ) + assert "must not access" in analysis["role_boundary"]["solver"] active_monitor = analysis["role_boundary"]["active_campaign_monitor"] assert "exact-job runtime" in active_monitor @@ -416,6 +447,847 @@ def _trajectory( } +def _task_environment_policy() -> dict[str, object]: + return { + "schema_version": BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION, + "policy_id": "fixture-task-submission", + "task_environment_access": { + "schema_version": BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "6" * 64, + "authority": "benchmark_task", + "purpose": "task_submission", + "namespace": "TASKCAP_", + "variable_names": [ + "TASKCAP_SUBMIT_PATHS", + "TASKCAP_JUDGE_URL", + "TASKCAP_SUBMIT_EXCLUDE_FLAGS", + "TASKCAP_TOKEN", + "TASKCAP_PATCH_DIR", + ], + }, + } + + +def _task_environment_read_trajectory( + values: dict[str, str], +) -> dict[str, object]: + trajectory = _trajectory() + tool_call = trajectory["steps"][0]["tool_calls"][0] + tool_call["function_name"] = BENCHMARK_TASK_ENVIRONMENT_READ_TOOL + tool_call["arguments"] = { + "schema_version": BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "6" * 64, + "variable_names": list(values), + } + trajectory["steps"][0]["observation"] = { + "schema_version": BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "6" * 64, + "values": values, + } + return trajectory + + +def _canonical_sha256(value: object) -> str: + encoded = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _declared_feedback_policy() -> dict[str, object]: + return { + "schema_version": BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION, + "policy_id": "fixture-declared-feedback", + "evaluation_protocol": { + "protocol_id": "fixture-declared-feedback-v1", + "feedback": "declared", + "timing": "during_solving", + "channels": [ + { + "channel_id": "result", + "phase": "solving", + "output_kinds": ["score", "validation"], + } + ], + }, + } + + +def _declared_feedback_attestation( + trajectory: dict[str, object], + policy: dict[str, object], +) -> dict[str, object]: + attestation = _attestation() + attestation["verifier_started_after_agent"] = False + attestation["official_feedback_blinded"] = False + attestation["verifier_started_after_submission"] = True + attestation["evaluation_protocol_enforced"] = True + attestation["evaluation_log_complete"] = True + submission = { + "submission_id": "submission-1", + "submission_sha256": "1" * 64, + "captured_at_ms": 200, + "evaluation_started_at_ms": 210, + "evaluation_completed_at_ms": 220, + } + delivery = { + "submission_id": "submission-1", + "channel_id": "result", + "phase": "solving", + "output_kinds": ["score", "validation"], + "feedback_sha256": "2" * 64, + "delivered_at_ms": 230, + } + evaluation_protocol = { + "run_id": "fixture-run", + "policy_sha256": _canonical_sha256(policy), + "trajectory_sha256": _canonical_sha256(trajectory), + "policy_admitted_at_ms": 100, + "solver_started_at_ms": 150, + "solver_ended_at_ms": 300, + "submissions": [submission], + "deliveries": [delivery], + } + attestation["evaluation_protocol"] = evaluation_protocol + attestation["feedback_transport"] = { + "run_id": "fixture-run", + "request_count": 1, + "response_ack_count": 1, + "intent_count": 1, + "ack_count": 1, + "deliveries_sha256": _canonical_sha256([delivery]), + "source_manifest_sha256": "3" * 64, + "record_digests": { + "send-1.json": "4" * 64, + "ack-1.json": "5" * 64, + }, + } + return attestation + + +def test_declared_during_solving_feedback_uses_protocol_attestations() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=_declared_feedback_attestation(trajectory, policy), + policy=policy, + ) + + assert receipt["integrity_qualified"] is True + assert receipt["runtime_attestation_checks"][ + "verifier_started_after_submission" + ] is True + assert receipt["runtime_attestation_checks"][ + "evaluation_protocol_enforced" + ] is True + assert "verifier_started_after_agent" not in receipt["runtime_attestation_checks"] + assert "official_feedback_blinded" not in receipt["runtime_attestation_checks"] + assert receipt["evaluation_protocol"] == { + "feedback": "declared", + "timing": "during_solving", + "policy_sha256": _canonical_sha256(policy), + "submission_count": 1, + "delivery_count": 1, + "evidence_sha256": _canonical_sha256( + { + "evaluation_protocol": _declared_feedback_attestation( + trajectory, policy + )["evaluation_protocol"], + "feedback_transport": _declared_feedback_attestation( + trajectory, policy + )["feedback_transport"], + } + ), + } + + +def test_declared_feedback_evidence_digest_binds_transport() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + first_attestation = _declared_feedback_attestation(trajectory, policy) + second_attestation = copy.deepcopy(first_attestation) + second_attestation["feedback_transport"]["source_manifest_sha256"] = "7" * 64 + + first = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=first_attestation, + policy=policy, + ) + second = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=second_attestation, + policy=policy, + ) + + assert first["integrity_qualified"] is True + assert second["integrity_qualified"] is True + assert ( + first["evaluation_protocol"]["evidence_sha256"] + != second["evaluation_protocol"]["evidence_sha256"] + ) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda runtime: runtime["submissions"].append( + { + "submission_id": "submission-out-of-order", + "submission_sha256": "8" * 64, + "captured_at_ms": 190, + "evaluation_started_at_ms": 240, + "evaluation_completed_at_ms": 250, + } + ), + lambda runtime: runtime["submissions"].extend( + [ + { + "submission_id": "submission-drain-not-final", + "submission_sha256": "8" * 64, + "captured_at_ms": 240, + "evaluation_started_at_ms": 290, + "evaluation_completed_at_ms": 310, + }, + { + "submission_id": "submission-after-drain", + "submission_sha256": "9" * 64, + "captured_at_ms": 250, + "evaluation_started_at_ms": 260, + "evaluation_completed_at_ms": 270, + }, + ] + ), + ], +) +def test_declared_feedback_rejects_unordered_or_nonfinal_drain_submissions( + mutation: Callable[[dict[str, object]], None], +) -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, policy) + mutation(attestation["evaluation_protocol"]) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_submissions_invalid" in receipt["blockers"] + + +def test_declared_feedback_allows_one_final_submission_to_drain() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, policy) + attestation["evaluation_protocol"]["submissions"].append( + { + "submission_id": "submission-final-drain", + "submission_sha256": "8" * 64, + "captured_at_ms": 290, + "evaluation_started_at_ms": 295, + "evaluation_completed_at_ms": 310, + } + ) + transport = attestation["feedback_transport"] + transport["request_count"] = 2 + transport["response_ack_count"] = 2 + transport["record_digests"].update( + { + "send-2.json": "8" * 64, + "ack-2.json": "9" * 64, + } + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evaluation_protocol"]["submission_count"] == 2 + + +def test_declared_feedback_requires_delivery_for_each_completed_submission() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, policy) + attestation["evaluation_protocol"]["submissions"].append( + { + "submission_id": "submission-withheld", + "submission_sha256": "8" * 64, + "captured_at_ms": 240, + "evaluation_started_at_ms": 250, + "evaluation_completed_at_ms": 260, + } + ) + transport = attestation["feedback_transport"] + transport["request_count"] = 2 + transport["response_ack_count"] = 2 + transport["record_digests"].update( + { + "send-2.json": "8" * 64, + "ack-2.json": "9" * 64, + } + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_deliveries_invalid" in receipt["blockers"] + + +def _final_drain_attestation( + trajectory: dict[str, object], + policy: dict[str, object], +) -> dict[str, object]: + attestation = _declared_feedback_attestation(trajectory, policy) + attestation["evaluation_protocol"]["submissions"][0].update( + captured_at_ms=290, + evaluation_started_at_ms=295, + evaluation_completed_at_ms=310, + ) + attestation["evaluation_protocol"]["deliveries"] = [] + attestation["feedback_transport"].update( + intent_count=0, + ack_count=0, + deliveries_sha256=_canonical_sha256([]), + ) + return attestation + + +@pytest.mark.parametrize("omit_delivery", [False, True]) +def test_cli_final_drain_allows_zero_feedback_but_requires_complete_delivery_evidence( + tmp_path: Path, omit_delivery: bool +) -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _final_drain_attestation(trajectory, policy) + if omit_delivery: + attestation["evaluation_protocol"]["deliveries"] = None + paths = [] + for name, value in ( + ("trajectory", trajectory), + ("policy", policy), + ("attestation", attestation), + ): + path = tmp_path / f"private-{name}.json" + path.write_text(json.dumps(value), encoding="utf-8") + paths.append(path) + completed = subprocess.run( + [ + str(REPO_ROOT / "scripts/loopx"), + "benchmark", + "integrity-qualification", + "--trajectory-json", + str(paths[0]), + "--policy-json", + str(paths[1]), + "--runtime-attestation-json", + str(paths[2]), + "--require-qualified", + "--format", + "json", + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + receipt = json.loads(completed.stdout) + assert completed.returncode == (1 if omit_delivery else 0) + assert receipt["integrity_qualified"] is (not omit_delivery) + assert str(tmp_path) not in completed.stdout + if not omit_delivery: + assert receipt["evaluation_protocol"]["submission_count"] == 1 + assert receipt["evaluation_protocol"]["delivery_count"] == 0 + + +@pytest.mark.parametrize("counter", ["intent_count", "ack_count"]) +@pytest.mark.parametrize("value", [-1, False, True, 0.0, None, "0", 1]) +def test_final_drain_rejects_invalid_or_invented_feedback_counters( + counter: str, value: object +) -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _final_drain_attestation(trajectory, policy) + attestation["feedback_transport"][counter] = value + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_transport_invalid" in receipt["blockers"] + + +def test_zero_feedback_does_not_waive_delivery_for_completed_submission() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _final_drain_attestation(trajectory, policy) + attestation["evaluation_protocol"]["submissions"][0][ + "evaluation_completed_at_ms" + ] = 299 + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_deliveries_invalid" in receipt["blockers"] + + +def test_task_authorization_does_not_rewrite_frozen_feedback_policy_digest() -> None: + trajectory = _task_environment_read_trajectory( + {"TASKCAP_TOKEN": "fixture-task-token-private-value"} + ) + frozen_policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, frozen_policy) + qualification_policy = copy.deepcopy(frozen_policy) + qualification_policy["task_environment_access"] = copy.deepcopy( + _task_environment_policy()["task_environment_access"] + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=qualification_policy, + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evaluation_protocol"]["policy_sha256"] == ( + _canonical_sha256(frozen_policy) + ) + assert receipt["task_environment_access"]["state"] == "authorized" + + +@pytest.mark.parametrize( + ("mutation", "expected_blocker"), + [ + ( + lambda attestation: attestation.__setitem__( + "evaluation_protocol_enforced", False + ), + "runtime_attestation_evaluation_protocol_enforced_missing", + ), + ( + lambda attestation: attestation["evaluation_protocol"].__setitem__( + "policy_sha256", "0" * 64 + ), + "evaluation_protocol_policy_mismatch", + ), + ( + lambda attestation: attestation["evaluation_protocol"][ + "deliveries" + ][0].__setitem__("channel_id", "undeclared"), + "evaluation_protocol_deliveries_invalid", + ), + ( + lambda attestation: attestation["feedback_transport"].__setitem__( + "ack_count", 0 + ), + "evaluation_protocol_transport_invalid", + ), + ], +) +def test_declared_feedback_protocol_fails_closed_on_incomplete_evidence( + mutation: Callable[[dict[str, object]], None], + expected_blocker: str, +) -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, policy) + mutation(attestation) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is False + assert expected_blocker in receipt["blockers"] + + +def test_task_scoped_submission_environment_is_authorized_by_task_contract() -> None: + private_task_token = "fixture-task-token-private-value" + receipt = build_benchmark_integrity_qualification( + trajectory=_task_environment_read_trajectory( + { + "TASKCAP_SUBMIT_PATHS": "src,include", + "TASKCAP_JUDGE_URL": "http://127.0.0.1:8080", + "TASKCAP_SUBMIT_EXCLUDE_FLAGS": "--fixture", + "TASKCAP_TOKEN": private_task_token, + "TASKCAP_PATCH_DIR": "/workspace/patches", + } + ), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evidence_counts"]["credential_probe"] == 0 + assert receipt["task_environment_access"] == { + "schema_version": BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, + "state": "authorized", + "authorization_id": "fixture-task-submission-v1", + "authorization_sha256": "6" * 64, + "authority": "benchmark_task", + "purpose": "task_submission", + "authorized_probe_count": 1, + "variable_name_count": 5, + "variable_names_sha256": ( + "6f4904e0ff7fd5b8e40d2210c060e9747630314fd9e52fec30867b041f3572a1" + ), + "raw_names_recorded": False, + "raw_values_recorded": False, + } + assert private_task_token not in json.dumps(receipt, sort_keys=True) + + +def test_exact_task_environment_variable_read_is_authorized() -> None: + private_task_token = "fixture-task-token-private-value" + receipt = build_benchmark_integrity_qualification( + trajectory=_task_environment_read_trajectory( + {"TASKCAP_TOKEN": private_task_token} + ), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evidence_counts"]["credential_probe"] == 0 + assert receipt["task_environment_access"]["authorized_probe_count"] == 1 + assert private_task_token not in json.dumps(receipt, sort_keys=True) + + +@pytest.mark.parametrize( + ("command", "observation"), + [ + ( + "env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "env | grep -i taskcap", + ( + "TASKCAP_TOKEN=fixture-task-token-private-value\n" + "MODEL_API_KEY=fixture-provider-key-private-value" + ), + ), + ( + "env | grep -i taskcap", + ( + "TASKCAP_TOKEN=fixture-task-token-private-value\n" + "taskcap_unapproved=fixture-private-value" + ), + ), + ( + "/usr/bin/printenv MODEL_API_KEY", + "fixture-provider-key-private-value", + ), + ( + "PATH=/tmp:$PATH env | grep -i taskcap", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "env() { printf 'TASKCAP_TOKEN=spoofed\\n'; }; env | grep -i taskcap", + "TASKCAP_TOKEN=spoofed", + ), + ( + "/usr/bin/env | grep -i taskcap", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "/usr/bin/env | /usr/bin/grep -i taskcap", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "function /usr/bin/env { printf 'TASKCAP_TOKEN=spoofed\\n'; }; " + "/usr/bin/env | /usr/bin/grep -i taskcap", + "TASKCAP_TOKEN=spoofed", + ), + ( + "command env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "exec env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs -n 1 env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs --process-slot-var SLOT env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs --max-lines env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs -L 1 env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs --eof env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs --replace env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "xargs -tn 1 env", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ( + "sh -c 'command env'", + "TASKCAP_TOKEN=fixture-task-token-private-value", + ), + ], +) +def test_task_environment_contract_does_not_waive_broad_or_out_of_scope_probes( + command: str, + observation: str, +) -> None: + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory(command=command, observation=observation), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert "credential_probe" in receipt["blockers"] + assert receipt["task_environment_access"]["authorized_probe_count"] == 0 + assert receipt["task_environment_access"]["state"] == "rejected" + + +def test_xargs_payload_named_env_is_not_classified_as_executed_utility() -> None: + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory( + command="xargs echo env", + observation="env", + ), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evidence_counts"]["credential_probe"] == 0 + assert receipt["task_environment_access"]["state"] == "declared" + + +def test_xargs_optional_max_lines_does_not_consume_separate_value() -> None: + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory( + command="xargs --max-lines 1 env", + observation="xargs: 1: No such file or directory", + ), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evidence_counts"]["credential_probe"] == 0 + + +def test_task_environment_contract_does_not_hide_provider_credential_value() -> None: + provider_credential = "fixture-provider-key-private-value" + receipt = build_benchmark_integrity_qualification( + trajectory=_task_environment_read_trajectory( + {"TASKCAP_TOKEN": provider_credential} + ), + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + sensitive_values=[provider_credential], + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 0 + assert receipt["evidence_counts"]["credential_value_observed"] == 1 + assert "credential_value_observed" in receipt["blockers"] + assert provider_credential not in json.dumps(receipt, sort_keys=True) + + +def test_task_environment_contract_does_not_mask_orchestrator_env_access() -> None: + trajectory = _trajectory() + tool_call = trajectory["steps"][0]["tool_calls"][0] + tool_call["function_name"] = "functions.exec" + tool_call["arguments"] = { + "input": ( + "const provider = process.env.MODEL_API_KEY; " + 'await tools.exec_command({cmd: "env | grep -i taskcap"});' + ) + } + trajectory["steps"][0]["observation"] = ( + "TASKCAP_TOKEN=fixture-task-token-private-value" + ) + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert receipt["task_environment_access"]["authorized_probe_count"] == 0 + + +def test_unauthorized_probe_dominates_authorized_task_environment_state() -> None: + trajectory = _task_environment_read_trajectory( + {"TASKCAP_TOKEN": "fixture-task-token-private-value"} + ) + trajectory["steps"].append( + _trajectory( + command="env", + observation="TASKCAP_TOKEN=fixture-task-token-private-value", + )["steps"][0] + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert receipt["task_environment_access"]["authorized_probe_count"] == 1 + assert receipt["task_environment_access"]["state"] == "rejected" + + +def test_task_environment_contract_requires_provider_credential_isolation() -> None: + attestation = _attestation() + attestation["provider_credential_shell_excluded"] = False + receipt = build_benchmark_integrity_qualification( + trajectory=_task_environment_read_trajectory( + {"TASKCAP_TOKEN": "fixture-task-token-private-value"} + ), + runtime_attestation=attestation, + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert "runtime_attestation_provider_credential_shell_excluded_missing" in ( + receipt["blockers"] + ) + + +@pytest.mark.parametrize( + ("argument_mutation", "observation_mutation"), + [ + ( + lambda arguments: arguments.__setitem__( + "authorization_sha256", "7" * 64 + ), + None, + ), + ( + lambda arguments: arguments.__setitem__( + "variable_names", ["TASKCAP_UNDECLARED"] + ), + None, + ), + ( + None, + lambda observation: observation["values"].__setitem__( + "MODEL_API_KEY", "fixture-provider-key-private-value" + ), + ), + ], +) +def test_task_environment_contract_rejects_mismatched_typed_read( + argument_mutation: Callable[[dict[str, object]], None] | None, + observation_mutation: Callable[[dict[str, object]], None] | None, +) -> None: + trajectory = _task_environment_read_trajectory( + {"TASKCAP_TOKEN": "fixture-task-token-private-value"} + ) + tool_call = trajectory["steps"][0]["tool_calls"][0] + if argument_mutation is not None: + argument_mutation(tool_call["arguments"]) + if observation_mutation is not None: + observation_mutation(trajectory["steps"][0]["observation"]) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert receipt["task_environment_access"]["state"] == "rejected" + + +@pytest.mark.parametrize( + "function_name", + [ + BENCHMARK_TASK_ENVIRONMENT_READ_TOOL.upper(), + f" {BENCHMARK_TASK_ENVIRONMENT_READ_TOOL} ", + ], +) +def test_task_environment_contract_requires_exact_typed_tool_name( + function_name: str, +) -> None: + trajectory = _task_environment_read_trajectory( + {"TASKCAP_TOKEN": "fixture-task-token-private-value"} + ) + trajectory["steps"][0]["tool_calls"][0]["function_name"] = function_name + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=_attestation(), + policy=_task_environment_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["credential_probe"] == 1 + assert receipt["task_environment_access"]["state"] == "rejected" + + +def test_task_environment_contract_requires_one_exact_namespace() -> None: + policy = _task_environment_policy() + task_environment_access = policy["task_environment_access"] + assert isinstance(task_environment_access, dict) + task_environment_access["variable_names"] = [ + "TASKCAP_TOKEN", + "UNSCOPED_TOKEN", + ] + + with pytest.raises( + ValueError, + match="^benchmark_task_environment_access_variable_outside_namespace$", + ): + build_benchmark_integrity_qualification( + trajectory=_trajectory(), + runtime_attestation=_attestation(), + policy=policy, + ) + + def _restricted_access_adjudication( *, disclosed: bool, causal_use: bool ) -> dict[str, object]: