Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ const fieldCopy: Record<WorkspaceLocale, FieldCopy> = {
executor_model: { label: "Model", description: "Optional model for the selected executor. Leave blank to keep the executor's own default." },
executor_reasoning_effort: { label: "Reasoning effort", description: "Optional reasoning effort for the selected executor. Leave blank to keep the executor's own default." },
count_unit: { label: "Review after", description: "Work Turns require accepted settlement; polls and retries do not count.", options: { completed_todos: "Completed Todos", effective_turns: "Settled work Turns" } },
planning_horizon_goal: { label: "Show Goal in planning horizon", description: "Default off. Read the full original Goal and acceptance before choosing work. Does not change replan, stride or execution authority." },
stride_strategy: { label: "Stride strategy", description: "Structured work scope, route and validation proposals are required for enabled fresh Vision reviews. Does not grant execution authority.", options: { off: "Off", evidence_conditioned_v1: "Evidence-conditioned · v1" } },
count: { label: "Number between reviews", description: "Settled work Turns: 1–6; completed Todos: 1–5. Goal overrides take precedence over device defaults." },
allowed_domains: { label: "Allowed responsibility domains", description: "Enter one bounded, public-safe domain per line." },
Expand Down Expand Up @@ -194,6 +195,7 @@ const fieldCopy: Record<WorkspaceLocale, FieldCopy> = {
executor_model: { label: "模型", description: "所选执行器使用的模型,可留空;留空表示沿用执行器自身的默认模型。" },
executor_reasoning_effort: { label: "推理档位", description: "所选执行器使用的推理档位,可留空;留空表示沿用执行器自身的默认档位。" },
count_unit: { label: "复核计数依据", description: "有效工作 Turn 须完成结算;轮询和重复重试不计数。", options: { completed_todos: "已完成 Todo", effective_turns: "已结算工作 Turn" } },
planning_horizon_goal: { label: "在规划视野中显示 Goal", description: "默认关闭。选择工作前读取完整原目标与验收要求,不改变复盘、步幅策略或执行权限。" },
stride_strategy: { label: "步幅策略", description: "启用后,新的 Vision 路线复盘须提交工作大小、路线及验证范围的结构化提案,不授予执行权限。", options: { off: "关闭", evidence_conditioned_v1: "依据证据调整 · v1" } },
count: { label: "两次复核间的数量", description: "已结算工作 Turn 为 1–6,已完成 Todo 为 1–5;Goal 显式设置优先于设备默认值。" },
allowed_domains: { label: "允许的职责域", description: "每行填写一个有边界、可公开的职责域。" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -379,8 +379,8 @@ function CapabilityCatalog({ callbacks, catalog, goalId, loading, notification,
</p> : null}
{selected.effective_value_policy === "goal_stride_over_live_review_cadence" ? <p>
{locale === "zh-CN"
? `复盘周期沿用${selected.machine_current ? "设备" : "产品"}默认;步幅策略由本 Goal 设置。`
: `Review interval follows ${selected.machine_current ? "device" : "product"} defaults; this Goal sets the stride strategy.`}
? `复盘周期沿用${selected.machine_current ? "设备" : "产品"}默认;规划选项由本 Goal 设置。`
: `Review interval follows ${selected.machine_current ? "device" : "product"} defaults; this Goal sets the planning options.`}
</p> : null}
<CapabilityConfigurationFields
disabled={editorBusy}
Expand Down
20 changes: 20 additions & 0 deletions benchmark/edgebench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,26 @@ other replan reasons may trigger sooner. Both runtime and worker-profile receipt
record the resolved cadence. Only newly launched attempts use the new default;
keep existing runs and archived settings pinned.

### Optional Goal in the planning horizon

For `heartbeat-resume` and `heartbeat-explore`, add `--planning-horizon-goal`
to include the full same-source original Goal and acceptance in preselection
planning context. `--no-planning-horizon-goal` explicitly disables it. Omission
retains the ordinary disabled treatment. Plain and native-Goal workers reject
this setting. It does not enable stride, Explore or TurnEnvelope.

The adapter writes the shared `execution_profile.planning_horizon_goal` through
`configure-goal`, checks the effective readback before work, and records an
explicit setting in both worker profile and runtime receipt. Restore follows
the same path: an explicit true/false overrides the donor setting. An omitted
setting encountering an enabled donor fails the readback instead of silently
labeling the run off. Historical receipts without this field remain absent;
report export rejects mismatched or malformed settings.

Compare on/off from the same qualified pre-replan checkpoint with equal remaining
budget, source and other settings. Input fidelity does not establish better
research or scores. See the [shared Goal configuration and rollback](../../docs/reference/protocols/quota-planning-horizon-v0.md#enable-read-back-and-disable).

### Optional structured stride strategy

Add `--stride-strategy evidence_conditioned_v1` to a new `heartbeat-resume`
Expand Down
8 changes: 8 additions & 0 deletions benchmark/edgebench/export_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,11 @@ def export(runs_root: Path, selection: list, output: Path, *, observed_at: str):
if row.get("runner_revision") != receipt["runner_commit"]:
raise ValueError("Board and runner revision disagree")
_validate_feedback_payload(receipt, profile)
if receipt.get("planning_horizon_goal") != profile.get("planning_horizon_goal"):
raise ValueError("Profile disagrees with runtime: planning_horizon_goal")
for source in (receipt, profile):
if "planning_horizon_goal" in source and not isinstance(source["planning_horizon_goal"], bool):
raise ValueError("planning_horizon_goal must be a boolean")
if receipt.get("stride_strategy") != profile.get("stride_strategy"):
raise ValueError("Profile disagrees with runtime: stride_strategy")
if "stride_strategy" in receipt:
Expand Down Expand Up @@ -329,6 +334,9 @@ def export(runs_root: Path, selection: list, output: Path, *, observed_at: str):
if "feedback_payload" in receipt:
config["runner"]["feedback_payload"] = receipt["feedback_payload"]
config["worker_profile"]["feedback_payload"] = profile["feedback_payload"]
if "planning_horizon_goal" in receipt:
config["runner"]["planning_horizon_goal"] = receipt["planning_horizon_goal"]
config["worker_profile"]["planning_horizon_goal"] = profile["planning_horizon_goal"]
if "stride_strategy" in receipt:
config["runner"]["stride_strategy"] = receipt["stride_strategy"]
config["worker_profile"]["stride_strategy"] = profile["stride_strategy"]
Expand Down
7 changes: 7 additions & 0 deletions benchmark/edgebench/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ def main(argv=None):
help="Heartbeat default: 6 settled effective work Turns")
cadence.add_argument("--replan-after-todos", type=int, choices=range(1, 6),
help="Explicit completed-Todo cadence ablation for heartbeat profiles")
parser.add_argument("--planning-horizon-goal", action=argparse.BooleanOptionalAction, default=None,
help="Opt-in full Goal in preselection horizon; --no-planning-horizon-goal explicitly disables it")
parser.add_argument("--stride-strategy", choices=stride_strategy_options(),
help="Explicit heartbeat Goal-review strategy; omitted keeps the ordinary off treatment")
parser.add_argument("--checkpoint-controller-file", type=Path,
Expand Down Expand Up @@ -168,6 +170,8 @@ def main(argv=None):
parser.error("Replan cadence requires a heartbeat worker")
if args.stride_strategy is not None and not args.worker.startswith("heartbeat-"):
parser.error("--stride-strategy requires a heartbeat worker")
if args.planning_horizon_goal is not None and not args.worker.startswith("heartbeat-"):
parser.error("--planning-horizon-goal requires a heartbeat worker")
checkpoint_request = None
if args.checkpoint_request is not None:
if not args.worker.startswith("heartbeat-"):
Expand Down Expand Up @@ -276,6 +280,7 @@ def main(argv=None):
native_goal_continue=args.native_goal_continue,
replan_after_turns=args.replan_after_turns,
replan_after_todos=args.replan_after_todos,
planning_horizon_goal=args.planning_horizon_goal,
**({"stride_strategy": args.stride_strategy}
if args.stride_strategy is not None else {}),
**({"checkpoint_request": checkpoint_request}
Expand Down Expand Up @@ -352,6 +357,8 @@ def main(argv=None):
if agent.replan_after_turns is not None else {}),
**({"replan_after_completed_todos": agent.replan_after_todos}
if agent.replan_after_todos is not None else {}),
**({"planning_horizon_goal": args.planning_horizon_goal}
if args.planning_horizon_goal is not None else {}),
**({"stride_strategy": args.stride_strategy}
if args.stride_strategy is not None else {}),
**({"checkpoint_request_sha256": hashlib.sha256(checkpoint_request).hexdigest()}
Expand Down
17 changes: 16 additions & 1 deletion benchmark/runtime/harbor.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def __init__(
replan_after_todos=None,
replan_after_turns=None,
stride_strategy=None,
planning_horizon_goal=None,
checkpoint_request_path=None,
checkpoint_request=None,
task_entry=None,
Expand Down Expand Up @@ -115,6 +116,12 @@ def __init__(
raise ValueError("stride_strategy requires a LoopX execution mode")
self.stride_strategy = (normalize_stride_strategy(stride_strategy)
if stride_strategy is not None else None)
if planning_horizon_goal is not None:
if not isinstance(planning_horizon_goal, bool):
raise ValueError("planning_horizon_goal must be a boolean")
if not self.execution.uses_loopx:
raise ValueError("planning_horizon_goal requires a LoopX execution mode")
self.planning_horizon_goal = planning_horizon_goal
self.checkpoint_request_path = None
self._checkpoint_declaration = None
if checkpoint_request is not None:
Expand Down Expand Up @@ -169,7 +176,9 @@ def _replan_configuration(self) -> tuple[str, str, int]:
def _replan_receipt(self) -> dict[str, int | str]:
key, _, value = self._replan_configuration()
return {key: value, **({"stride_strategy": self.stride_strategy}
if self.stride_strategy is not None else {})}
if self.stride_strategy is not None else {}),
**({"planning_horizon_goal": self.planning_horizon_goal}
if self.planning_horizon_goal is not None else {})}

@staticmethod
def name() -> str:
Expand Down Expand Up @@ -422,6 +431,9 @@ async def _prepare_phase(
key, option, expected = self._replan_configuration()
stride_args = (["--execution-stride-strategy", self.stride_strategy]
if self.stride_strategy is not None else [])
if self.planning_horizon_goal is not None:
stride_args.append("--execution-planning-horizon-goal" if self.planning_horizon_goal
else "--no-execution-planning-horizon-goal")
pending = await environment.exec(
command=f"test -e {_LOOPX_RUNTIME}/benchmark-pending-turn.json"
)
Expand Down Expand Up @@ -502,6 +514,9 @@ async def _prepare_phase(
raise RuntimeError(
f"replan cadence readback mismatch for {key}: expected {expected}, got {configured!r}"
)
actual_horizon = configured_state.get("execution_profile", {}).get("planning_horizon_goal", False)
if actual_horizon != (self.planning_horizon_goal or False):
raise RuntimeError("planning_horizon_goal readback mismatch")
strategy = configured_state.get("execution_profile", {}).get("stride_strategy", "off")
if strategy != (self.stride_strategy or "off"):
raise RuntimeError(
Expand Down
8 changes: 8 additions & 0 deletions benchmark/runtime/sforge.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ def __init__(self, config, *, profile: str, cwd: str,
replan_after_turns: int | None = None,
replan_after_todos: int | None = None,
stride_strategy: str | None = None,
planning_horizon_goal: bool | None = None,
checkpoint_request_path: str | None = None,
checkpoint_request: bytes | None = None,
warm_start=None):
Expand Down Expand Up @@ -143,6 +144,12 @@ def __init__(self, config, *, profile: str, cwd: str,
if stride_strategy is not None:
from loopx.control_plane.work_items.stride_policy_codec import normalize_stride_strategy
stride_strategy = normalize_stride_strategy(stride_strategy)
if planning_horizon_goal is not None:
if not isinstance(planning_horizon_goal, bool):
raise ValueError("planning_horizon_goal must be a boolean")
if not profile.startswith("heartbeat-"):
raise ValueError("planning_horizon_goal requires a heartbeat profile")
self.planning_horizon_goal = planning_horizon_goal
self.stride_strategy = stride_strategy
self.checkpoint_request_path = None
self.checkpoint_request = checkpoint_request
Expand Down Expand Up @@ -265,6 +272,7 @@ def _install_worker(self, backend, handle, log_dir, logger):
replan_after_turns=self.replan_after_turns,
replan_after_todos=self.replan_after_todos,
stride_strategy=self.stride_strategy,
planning_horizon_goal=self.planning_horizon_goal,
checkpoint_request_path=self.checkpoint_request_path,
checkpoint_request=self.checkpoint_request,
task_objective=self.task_objective,
Expand Down
29 changes: 29 additions & 0 deletions benchmark/tests/test_edgebench_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -625,3 +625,32 @@ def build(repository):
return data

_publish_and_recheck(tmp_path, build)


@pytest.mark.parametrize("enabled", [None, False, True])
def test_report_preserves_horizon_opt_in_and_historical_absence(trial, enabled):
runs, selection, output, _ = trial
for name in ("runtime-receipt", "worker-profile"):
path = runs / "run" / "case" / (name + ".json")
value = json.loads(path.read_text())
if enabled is not None:
value["planning_horizon_goal"] = enabled
write(path, value)
export(runs, selection, output, observed_at="2026-01-02T00:00:00+00:00")
config = json.loads((output / "settings/run.json").read_text())
for field in ("runner", "worker_profile"):
assert config[field].get("planning_horizon_goal") is enabled
assert ("planning_horizon_goal" in config[field]) is (enabled is not None)


@pytest.mark.parametrize("receipt,profile", [(True, False), (True, None), ("true", "true"), (True, 1)])
def test_report_rejects_unbound_horizon_setting(trial, receipt, profile):
runs, selection, output, _ = trial
for name, enabled in (("runtime-receipt", receipt), ("worker-profile", profile)):
path = runs / "run" / "case" / (name + ".json")
value = json.loads(path.read_text())
if enabled is not None:
value["planning_horizon_goal"] = enabled
write(path, value)
with pytest.raises(ValueError, match="planning_horizon_goal"):
export(runs, selection, output, observed_at="2026-01-02T00:00:00+00:00")
21 changes: 18 additions & 3 deletions benchmark/tests/test_sforge_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,8 @@ def test_planned_sforge_entry_rejects_unsupported_profiles(profile, monkeypatch)


@pytest.mark.parametrize("entry", ["seeded-todo", "loopx-planned"])
def test_sforge_command_preserves_native_timing_and_planning_boundary(tmp_path, monkeypatch, entry):
@pytest.mark.parametrize("horizon", [None, False, True])
def test_sforge_command_preserves_native_timing_and_planning_boundary(tmp_path, monkeypatch, entry, horizon):
pytest.importorskip("sforge")
pytest.importorskip("harbor")
from sforge.harness.config import SForgeConfig
Expand All @@ -472,17 +473,22 @@ async def installed(self, environment):
pass
monkeypatch.setattr(BenchmarkCodex, "install", installed)
worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="xhigh"),
profile="heartbeat-explore", cwd="/task", task_entry=entry)
profile="heartbeat-explore", cwd="/task", task_entry=entry, planning_horizon_goal=horizon)
worker.install_stop_hook(None, None, tmp_path, None)
assert worker.runtime.execution.task_entry == entry
assert worker.runtime.planning_horizon_goal is horizon
assert worker.runtime.execution.turn_envelope is False
worker.prepared = True # Bootstrap is separately covered through real CLI fixtures.
first = worker.format_run_cmd("/task.md")
resumed = worker.format_run_cmd("/task.md", resume=True)
assert first == resumed
assert ("benchmark.runtime.sforge_entry" in first) is (entry == "loopx-planned")
assert "test -f /opt/loopx-benchmark/control/phase-deadline ||" in first
assert first.index("export LOOPX_PHASE_DEADLINE_EPOCH") < first.index("timeout --signal")
assert json.loads((tmp_path / "worker-profile.json").read_text())["task_entry"] == entry
profile = json.loads((tmp_path / "worker-profile.json").read_text())
assert profile["task_entry"] == entry
assert profile.get("planning_horizon_goal") is horizon
assert ("planning_horizon_goal" in profile) is (horizon is not None)
@pytest.mark.parametrize('profile', ['heartbeat-resume', 'heartbeat-explore'])
@pytest.mark.parametrize('enabled', [False, True])
@pytest.mark.parametrize('cadence', [None, 2, 6])
Expand Down Expand Up @@ -656,6 +662,8 @@ def test_invalid_checkpoint_declaration_rejects_before_trial_registration(tmp_pa

@pytest.mark.parametrize("cadence_args,field,count", [
([], "replan_after_effective_turns", 6),
(["--planning-horizon-goal"], "replan_after_effective_turns", 6),
(["--no-planning-horizon-goal"], "replan_after_effective_turns", 6),
(["--replan-after-turns", "2"], "replan_after_effective_turns", 2),
(["--replan-after-todos", "3"], "replan_after_completed_todos", 3),
])
Expand Down Expand Up @@ -693,6 +701,9 @@ def stop_before_solver(**kwargs):
assert kwargs["timeout"] == kwargs["config"].agent_timeout == expected
assert kwargs["eval_interval"] == interval
agent = kwargs["agent"]
horizon = (True if "--planning-horizon-goal" in cadence_args else
False if "--no-planning-horizon-goal" in cadence_args else None)
assert agent.planning_horizon_goal is horizon
assert "total trial budget" not in agent.workspace_instructions
assert agent.task_objective == "fixture task"
assert agent.task_objective in agent.feedback_prompt
Expand All @@ -713,6 +724,10 @@ def stop_before_solver(**kwargs):
assert receipt["timeout_seconds"] == expected
assert receipt["eval_interval"] == interval
assert receipt[field] == count
horizon = (True if "--planning-horizon-goal" in cadence_args else
False if "--no-planning-horizon-goal" in cadence_args else None)
assert receipt.get("planning_horizon_goal") is horizon
assert ("planning_horizon_goal" in receipt) is (horizon is not None)
other = ("replan_after_completed_todos" if field == "replan_after_effective_turns"
else "replan_after_effective_turns")
assert other not in receipt
Expand Down
Loading
Loading