Repository navigation
Expand file tree
/
Copy pathrelease.json
More file actions
60 lines (60 loc) · 2.37 KB
/
Copy pathrelease.json
File metadata and controls
60 lines (60 loc) · 2.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
{
"name": "PersonaCodeBench Lite",
"version": "0.5.0rc9",
"schema_version": "1.0.0",
"distribution": "lite",
"dataset": {
"file": "personacodebench-lite.parquet",
"tasks": 100,
"unique_task_ids": 100,
"sha256": "1fbfa744b505609e02d83f9a7d7bacb41b3f72777185f131bcdc7aeb10c460ed",
"bytes": 114985524,
"source_benchmarks": {
"Terminal-Bench 3.0": 23,
"SWE-bench Pro": 26,
"SWE-bench-Live MultiLang": 19,
"SWE-Together": 4,
"DeepSWE": 28
}
},
"runtime_profiles": {
"terminal-bench-3": 23,
"swe-bench-pro-mini": 9,
"swe-bench-pro-shell": 17,
"swe-bench-live": 19,
"swe-together": 4,
"deep-swe": 28
},
"runtime_sha256": "8219ddf5151c86f363d3e67004b438765002d7f2bf60a8a6b3ec17b5ea6835f6",
"scoring_entrypoint": "scripts/score.py",
"evaluation_entrypoint": "personacodebench run",
"batch_entrypoint": "scripts/evaluate.py",
"all_tasks_end_to_end_certified": false,
"api_configuration_roles": [
"agent",
"user_simulator"
],
"judge_api_source": "user_simulator",
"native_stability_entrypoint": "scripts/check_live_native.py",
"api_output_recovery_policy": "bounded-output-recovery-v1",
"simulator_output_protocol": "simulator-json-envelope-v1",
"agent_output_protocol": "coding-agent-json-envelope-v1",
"shell_clause_protocol": "literal-clause-truth-v1",
"shell_calibration_fixture": "packaged-rubric-control-v1",
"structured_output_fallback_policy": "structured-output-fallback-v1",
"together_calibration_fixture": "swe-together-calibration-fixtures-v3",
"together_simulator_transport": "strict-decision-schema-v1",
"shell_native_failure_policy": "reference-execution-control-v1",
"live_test_projection": "cwd-independent-official-test-files-v1",
"shell_native_reference_controls": "swepro-native-reference-controls-v1",
"together_overall_judge_prompt": "overall_completion_v3",
"together_interaction_stall_policy": "silent-no-progress-v2",
"together_interaction_stall_default_turns": 4,
"together_calibration_gate": "swe-together-calibration-gate-v1",
"together_calibration_default_mode": "bounded",
"together_calibration_max_reviewed_positive_disagreements": 2,
"time_budget_policy": "agent-user-wall-clock-v2",
"preparation_timeout_default_sec": 7200,
"verification_watchdog_default": "max(900, budget.verifier_timeout_sec + 600)",
"calibration_default_enabled": true
}