diff --git a/doc/architecture/SPEC/config.md b/doc/architecture/SPEC/config.md index 75775a70..f74ebbfe 100644 --- a/doc/architecture/SPEC/config.md +++ b/doc/architecture/SPEC/config.md @@ -1,6 +1,6 @@ # config.py —— 规范 - + `LOC ~658 · 配置 · refactor-status: oversized` @@ -101,6 +101,8 @@ provider 的缺省模型必须在**这一处**解析，否则 `ResolvedTarget.mo `TRACE_STORE_ROOT` 控制 trace/1 全量采集。`IMPROVE_ENABLED=false` 是总停止开关，`IMPROVE_EVOLVE_ENABLED=false` 控制代码进化；源码、数据和发布 outbox 各有独立目录。周预算默认 20 美元，周度槽位默认周一 05:00 UTC。`PR_CONTEXT_SOURCE=snapshot` 与 `IMPROVE_SHADOW` 由可信实验控制器设置；`KB_DRAFT_MAX_OPERATIONS` 默认 6。 +`IMPROVE_INPUT_CONTEXT_LIMITS` 默认 `{}`，按精确请求模型名配置正整数输入上下文上限；空模型名、布尔值及非正整数均拒绝。网关可能补充不可见输入时，每次调用按该上限与可见请求字节数的较大值预留，实验子进程继承父 governor 的映射，候选覆盖无效。美元及判官周预算、实际用量结算与超预留中止门不变。 + ## 自主进化接入 自主进化新增 `improve_evaluation_mode=objective`、`improve_promotion_mode=automatic`、`improve_evolve_model`；固定单模型，无需判官。总开关与周预算默认不变。 diff --git a/doc/architecture/SPEC/improve.md b/doc/architecture/SPEC/improve.md index 491da976..3ee21694 100644 --- a/doc/architecture/SPEC/improve.md +++ b/doc/architecture/SPEC/improve.md @@ -1,6 +1,6 @@ # improve/ —— 规范（元改进引擎） - + `设计：/data/zhoutaichang/copilot/meta-improvement-engine-design.md v1（GPT-6 sol 批准 2026-09-29） · refactor-status: building (P0–P4 已落地)` @@ -39,7 +39,7 @@ | `workflows/kb-intake.yaml` | P2+ | 起草步骤的声明：static/step_call，item `{repo}#{pr}`，指纹覆盖 `kb_service/intake.py`、`KB_GENERATOR`/`KB_DRAFT_STRATEGY`、`zcode_reasoning_level`、copilot_sha；`kb_service.runtime` 为每次起草盖上 workflow/unit_id/item/fingerprint | | `forensics.py` | P2 | 覆盖矩阵（确定性）、S0–S10 阶段分类法、取证 agent（只读 trace 工具、全部输出围栏为不可信数据、必须引用记录 id）、双家族交叉复核（不一致 = disputed）、整改清单、测量健康 | | `meta.py` | P2 | 冻结元基准：`eval/dataset/meta/cases`（trace + 人工阶段标签）与 `meta/lints`（注入缺陷样本）的导出与加载 | -| `budget.py` | P3 | 周包络（美元 + 判官次数，按 ISO 周持久化、跨进程加锁）：每次模型调用发出前按最坏情况预留（输入 = 请求字节数 ≥ token 数、输出 = `max_tokens`、无价格即拒绝）、返回后结算；`结算 > 预留` 记 `budget_breach` 并中止；`governed()`/`current_governor()` 供 `LLM.create` 与 `run_judge` 使用 | +| `budget.py` | P3 | 周包络（美元 + 判官次数，按 ISO 周持久化、跨进程加锁）：每次模型调用发出前按最坏情况预留（输入 = 可见请求字节数与精确模型上下文上限的较大值，按最高输入／缓存单价计；输出 = `max_tokens`；无价格即拒绝）、返回后按实际用量结算；`结算 > 预留` 记 `budget_breach` 并中止；`governed()`/`current_governor()` 供 `LLM.create` 与 `run_judge` 使用 | | `experiments.py` | P3 | 预注册（Tier 2 且非 descriptive-only、API 后端、指纹 diff 非空且不触及元基准、每个 item 有金标、按历史 sd 算 `n_required`、成本预留）→ 影子运行（stage → 每 item×replicate×臂 一个子进程、`PR_SNAPSHOT_FILE` 交接、指纹核对、L13/预算/崩溃隔离）→ 配对判定（`n_retained` 重算功效、五种标签、`experiment_verdict` 记录、提案状态推进） | | `publish.py` | P4 | 提案发布：引用只以记录 id + blob 哈希（`excerpt_for` 逐字摘录 ≤20 行、脱敏、`verify_excerpt` 可由哈希复原）；`lint_proposal` 拒绝任何不可解析引用（Tier 1 ≥3 条记录，Tier 2 S1–S9 + 损失量）；固定 issue 模板（主张/阶段/损失/证据/建议预注册/账本/marker）；`ProposalOutbox` 文件协议 `improve-outbox/1`（`actions/` 引擎写、`acks/` 与 `inbox/` maintainer routine 写）；`plan`/`publish`（open/update/close，hold 暂停发布、待 ack 不重发）；`sync`（ack → issue 与已发布状态；inbox → 人类触碰、`maintainer: hold`、合并 PR → landed、GitHub 关闭 → closed；Tier 1 landed 在后一周期 lint 率低于开单时才 close） | | `adapters/meta_bench.py` | P4 | 引擎自身的结果适配器（§11.2）：金标 = 元基准 case 的人工阶段标签；outcome = `meta_eval` 记录；hit = 标签一致、disputed = 双家族不一致、S0 = unlabeled；`review_scores` = κ、lint 召回；`human_labelled`（无判官） | @@ -60,8 +60,9 @@ - 既有提案的外向写是 `improve.publish`，代码候选另由 `improve.evolve_publish` 经 post/push 双门写独立 outbox；引擎不持有 GitHub 令牌，issue 的开/改/关由 maintainer routine 执行并以 ack 回报。 - 未通过 `lint_proposal` 的提案永不发布（记 `proposal_lint_failed`）；hold 只暂停发布，lint 与实验照常。 - 元基准只读：任何触及 `eval/dataset/meta` 的指纹 diff 或 `IMPROVE_*` 覆盖在注册时被拒。 +- 实验子进程的周预算、账本目录和 `IMPROVE_INPUT_CONTEXT_LIMITS` 由父 governor 固定，不接受候选或运行臂自己的配置覆盖。修复估算后操作员可归档并清除已核对的预留故障，实际结算金额不回退。 - 引擎自身的记录（`playbook=workflow-improve`）是下一周期的 Tier 1 单元（自登记）。 - lint 只读 trace，不调模型；无证据形态的 lint 不猜。 ## 测试 -`test_improve_p0.py`、`test_improve_p0b.py`、`test_improve_p1.py`、`test_improve_p2.py`、`test_improve_p3.py`、`test_improve_p4.py`、`test_improve_kb_intake_adapter.py`、`test_evolution.py`、`test_autonomous_evolution.py`。 +`test_improve_p0.py`、`test_improve_p0b.py`、`test_improve_p1.py`、`test_improve_p2.py`、`test_improve_p3.py`、`test_improve_p4.py`、`test_improve_gateway_budget.py`、`test_improve_kb_intake_adapter.py`、`test_evolution.py`、`test_autonomous_evolution.py`。 diff --git a/doc/guide/self-evolution.md b/doc/guide/self-evolution.md index 74e6403b..185e21a4 100644 --- a/doc/guide/self-evolution.md +++ b/doc/guide/self-evolution.md @@ -4,6 +4,16 @@ 进化总开关仍默认关闭。每周最多一个新候选，开发检查失败最多修复一次。候选生成、实验、隔离生产请求及影子比较共用每周 20 美元预算；不足时延期或恢复上一版，不自行增加预算。 +兼容 API 网关可能添加不可见的系统提示，返回的输入 token 数可以超过请求正文的字节数。 +为这种模型配置服务商声明的完整上下文上限，用于每次调用前的保守费用预留，例如 +`IMPROVE_INPUT_CONTEXT_LIMITS='{"mimo-v2.5":1048576}'`。 +[MiMo-V2.5 的官方规格](https://mimo.mi.com/models/en-US/mimo-v2.5)声明 1M 上下文； +这里预留整个上下文的输入费用加请求的最大输出费用，实际结算仍使用服务商返回的用量。 +映射按请求模型名称精确匹配，并由父进程原样传给实验子进程，候选不能覆盖。 +这不会增加每周费用或判官调用上限。已发生的超预留仍会中止并保留故障记录；修复估算后， +操作员可先归档故障与原因，再用 `Governor.clear_breach()` 清除已核对的预留故障，已结算费用不变。 +预算和候选次数按 UTC ISO 周计算；上海时区的每周定时任务应安排在周一 08:00 之后，例如 09:00。 + ## 开始使用 从干净、已提交的 Copilot 源码运行，安装仓库的固定依赖： diff --git a/src/infermatrix_copilot/config.py b/src/infermatrix_copilot/config.py index 8a87437e..9fe245cd 100644 --- a/src/infermatrix_copilot/config.py +++ b/src/infermatrix_copilot/config.py @@ -610,6 +610,20 @@ def _parse_mixture(cls, v): # the hard weekly envelope (P3 enforces it per model call) improve_budget_usd_week: float = 20.0 improve_budget_judge_calls_week: int = 300 + # Exact requested model -> provider context ceiling. Reserve this many + # input tokens when gateways add hidden prompts beyond the visible bytes. + improve_input_context_limits: dict[str, int] = {} + + @field_validator("improve_input_context_limits", mode="before") + @classmethod + def _valid_improve_input_context_limits(cls, value): + if not isinstance(value, dict) or any( + not isinstance(model, str) or not model.strip() or type(limit) is not int or limit <= 0 + for model, limit in value.items() + ): + raise ValueError("improve input context limits require exact model names and positive integer ceilings") + return value + # the eval review adapter's inputs (design §6.1): the GT directory holding # gt/curated/.gold.json, the judgment sets, and the arm name whose # verdicts belong to this copilot's units diff --git a/src/infermatrix_copilot/improve/budget.py b/src/infermatrix_copilot/improve/budget.py index 47fcec39..8993ccd8 100644 --- a/src/infermatrix_copilot/improve/budget.py +++ b/src/infermatrix_copilot/improve/budget.py @@ -15,8 +15,10 @@ The reservation is a true upper bound: output tokens are the request's ``max_tokens`` (the API enforces it); input tokens — including cache reads and cache writes, which are prompt tokens billed at their own rates — are -bounded by the request's UTF-8 byte count (every BPE-family token is at -least one byte) priced at the cache-write rate (the dearest of the three). +bounded by the request's UTF-8 byte count for the visible payload. Gateways +can inject hidden prompts; configure their exact models in +``improve_input_context_limits`` with the provider's full context ceiling. +The larger bound is priced at the dearest configured input/cache rate. Settlement charges the cache-aware cost exactly as `metrics.cost_from_spans` does. Prices come from the metrics price table; a model with no price cannot be reserved and is refused, never priced at zero. If a settlement ever @@ -71,12 +73,19 @@ def request_bytes(kwargs: dict) -> int: def worst_case_usd(model: str, req_bytes: int, max_tokens: int, settings: Any = None) -> float: - from ..metrics import CACHE_CREATE_FACTOR, model_price, model_price_known + from ..metrics import CACHE_CREATE_FACTOR, CACHE_READ_FACTOR, model_price, model_price_known if not model_price_known(model, settings): raise BudgetRefused(f"no price for model {model!r}: it cannot be reserved (never priced at zero)") pin, pout = model_price(model, settings) - return req_bytes / 1e6 * pin * CACHE_CREATE_FACTOR + max(0, int(max_tokens or 0)) / 1e6 * pout + limits = getattr(settings, "improve_input_context_limits", {}) or {} + limit = limits.get(model, 0) + if type(limit) is not int or limit < 0: + raise BudgetRefused(f"invalid input context ceiling for model {model!r}") + input_bound = max(req_bytes, limit) + read_factor = float(getattr(settings, "cache_read_price_factor", 0) or 0) or CACHE_READ_FACTOR + input_factor = max(1.0, CACHE_CREATE_FACTOR, read_factor) + return input_bound / 1e6 * pin * input_factor + max(0, int(max_tokens or 0)) / 1e6 * pout def actual_usd(model: str, usage: dict | None, settings: Any = None) -> float: diff --git a/src/infermatrix_copilot/improve/experiments.py b/src/infermatrix_copilot/improve/experiments.py index 38d220aa..ea76cf36 100644 --- a/src/infermatrix_copilot/improve/experiments.py +++ b/src/infermatrix_copilot/improve/experiments.py @@ -530,15 +530,18 @@ def _run_locked(store, settings, ledger_dir: Path, exp: Experiment, *, run_unit, scores: dict[str, dict[str, dict[int, float]]] = {} # item -> side -> replicate -> metric value units_by_key: dict[str, str] = {} produced: dict[str, dict[str, dict[int, Unit]]] = {} # item -> side -> replicate -> unit + input_limits = getattr(governor.settings, "improve_input_context_limits", {}) or {} budget_env = {"IMPROVE_LEDGER_DIR": str(governor.dir.parent), "IMPROVE_BUDGET_USD_WEEK": str(governor.usd_week), - "IMPROVE_BUDGET_JUDGE_CALLS_WEEK": str(governor.judge_calls_week)} + "IMPROVE_BUDGET_JUDGE_CALLS_WEEK": str(governor.judge_calls_week), + "IMPROVE_INPUT_CONTEXT_LIMITS": json.dumps(input_limits, sort_keys=True)} for item, snap in staged.items(): repo, pr = snap["repo"], int(snap["pr"]) for side, overrides, expected_fp in (("arm", exp.arm_overrides, exp.arm_fingerprint), ("incumbent", exp.incumbent_overrides, exp.incumbent_fingerprint)): env = shadow_env(shadow_dir=snap["shadow_dir"], run_dir=run_root, trace_root=shadow_store.root, executables_dir=exe_dir, repo_name=repo, ledger_dir=governor.dir.parent, - usd_week=governor.usd_week, judge_calls_week=governor.judge_calls_week) + usd_week=governor.usd_week, judge_calls_week=governor.judge_calls_week, + input_context_limits=input_limits) env.update({k: str(v) for k, v in overrides.items()}) problems = assert_boundaries(env, expected=budget_env) if problems: diff --git a/src/infermatrix_copilot/improve/shadow.py b/src/infermatrix_copilot/improve/shadow.py index 134a66ab..865abe5d 100644 --- a/src/infermatrix_copilot/improve/shadow.py +++ b/src/infermatrix_copilot/improve/shadow.py @@ -110,7 +110,7 @@ def make_executables_dir(dest: str | Path, executables: Iterable[str] = DEFAULT_ def shadow_env(*, shadow_dir: str | Path, run_dir: str | Path, trace_root: str | Path, executables_dir: str | Path, repo_name: str, environ: dict | None = None, ledger_dir: str | Path | None = None, usd_week: float = 20.0, - judge_calls_week: int = 300) -> dict[str, str]: + judge_calls_week: int = 300, input_context_limits: dict[str, int] | None = None) -> dict[str, str]: """The allowlisted environment for a shadow subprocess (layer 3). The child binds its own budget governor from IMPROVE_GOVERNED=1 and the ledger directory, against the same weekly files as the parent.""" @@ -138,6 +138,7 @@ def shadow_env(*, shadow_dir: str | Path, run_dir: str | Path, trace_root: str | out["IMPROVE_LEDGER_DIR"] = str(ledger_dir) out["IMPROVE_BUDGET_USD_WEEK"] = str(usd_week) out["IMPROVE_BUDGET_JUDGE_CALLS_WEEK"] = str(judge_calls_week) + out["IMPROVE_INPUT_CONTEXT_LIMITS"] = json.dumps(input_context_limits or {}, sort_keys=True) return out diff --git a/test/test_improve_gateway_budget.py b/test/test_improve_gateway_budget.py new file mode 100644 index 00000000..93a54f6b --- /dev/null +++ b/test/test_improve_gateway_budget.py @@ -0,0 +1,78 @@ +"""Gateway-injected input is reserved without widening the weekly envelope.""" + +import json +import pytest + +from infermatrix_copilot.config import Settings +from infermatrix_copilot.improve.budget import BudgetBreach, BudgetRefused, Governor, worst_case_usd +from infermatrix_copilot.improve.shadow import assert_boundaries, shadow_env + + +LIMITS = {"mimo-v2.5": 1048576} +USAGE = {"input_tokens": 11430, "cache_read_input_tokens": 1664, "output_tokens": 16000} + + +def test_observed_gateway_usage_settles_inside_configured_context_reservation(tmp_path): + cfg = Settings(_env_file=None, improve_input_context_limits=LIMITS) + gov = Governor(tmp_path, usd_week=20, judge_calls_week=300, settings=cfg) + reservation = gov.reserve_call("mimo-v2.5", 5871, 16000) + assert gov.remaining()["usd_reserved"] == pytest.approx(0.481152) + actual = gov.settle_call(reservation, USAGE, "mimo-v2.5") + assert actual == pytest.approx(0.02645874) + assert gov.remaining()["usd_settled"] == pytest.approx(actual, abs=1e-6) + assert not gov.remaining()["breached"] + + +def test_gateway_reservation_refuses_before_dispatch_when_existing_cap_is_too_small(tmp_path): + cfg = Settings(_env_file=None, improve_input_context_limits=LIMITS) + gov = Governor(tmp_path, usd_week=0.1, judge_calls_week=300, settings=cfg) + with pytest.raises(BudgetRefused, match="exceed the weekly envelope"): + gov.reserve_call("mimo-v2.5", 5871, 16000) + assert gov.remaining()["usd_settled"] == 0 + assert gov.remaining()["usd_reserved"] == 0 + assert gov.remaining()["usd_envelope"] == 0.1 + + +def test_limits_are_exact_model_matches_and_never_reduce_visible_bound(): + cfg = Settings(_env_file=None, improve_input_context_limits=LIMITS) + assert worst_case_usd("mimo-v2.5-pro", 5871, 16000, cfg) == worst_case_usd("mimo-v2.5-pro", 5871, 16000) + assert worst_case_usd("mimo-v2.5", 2_000_000, 16000, cfg) == worst_case_usd("mimo-v2.5", 2_000_000, 16000) + cfg.cache_read_price_factor = 2.0 + assert worst_case_usd("mimo-v2.5", 5871, 16000, cfg) == pytest.approx(0.7564032) + + +@pytest.mark.parametrize("limits", [{"": 1}, {"mimo-v2.5": True}, {"mimo-v2.5": 0}, + {"mimo-v2.5": -1}, {"mimo-v2.5": "1000000"}, {"mimo-v2.5": 1.5}, []]) +def test_invalid_context_ceiling_is_rejected_at_configuration(limits): + with pytest.raises(ValueError, match="context limits"): + Settings(_env_file=None, improve_input_context_limits=limits) + + +def test_governed_child_receives_parent_limits_and_rejects_override(tmp_path, monkeypatch): + env = shadow_env(shadow_dir=tmp_path, run_dir=tmp_path, trace_root=tmp_path, + executables_dir=tmp_path, repo_name="demo", + ledger_dir=tmp_path, + environ={"IMPROVE_INPUT_CONTEXT_LIMITS": '{"mimo-v2.5":1}'}, + input_context_limits=LIMITS) + expected = {"IMPROVE_INPUT_CONTEXT_LIMITS": json.dumps(LIMITS, sort_keys=True)} + assert not assert_boundaries(env, expected=expected) + for key, value in env.items(): + monkeypatch.setenv(key, value) + assert Settings(_env_file=None).improve_input_context_limits == LIMITS + env["IMPROVE_INPUT_CONTEXT_LIMITS"] = "{}" + assert assert_boundaries(env, expected=expected) + + +def test_reconciling_an_estimate_breach_preserves_actual_charges_and_weekly_cap(tmp_path): + gov = Governor(tmp_path, usd_week=20, judge_calls_week=300) + reservation = gov.reserve_call("mimo-v2.5", 5871, 16000) + with pytest.raises(BudgetBreach): + gov.settle_call(reservation, USAGE, "mimo-v2.5") + before = gov.remaining() + gov.settings = Settings(_env_file=None, improve_input_context_limits=LIMITS) + gov.clear_breach() + after = gov.remaining() + assert after["usd_settled"] == before["usd_settled"] + assert after["usd_envelope"] == before["usd_envelope"] == 20 + assert after["usd_reserved"] == 0 + assert not after["breached"] diff --git a/test/test_improve_p3.py b/test/test_improve_p3.py index b2c79e71..cb31f5e8 100644 --- a/test/test_improve_p3.py +++ b/test/test_improve_p3.py @@ -423,7 +423,9 @@ def wrapped(**kw): assert len(verdicts) == 48 and {v["result"]["blinded_as"] for v in verdicts} == {"X", "Y"} # the child's budget is pinned to the governor even when settings name another directory other = TraceStore(tmp_path / "shadow-2", environ={}) - st2 = st.model_copy(update={"improve_ledger_dir": str(tmp_path / "elsewhere")}) + holder.settings = st.model_copy(update={"improve_input_context_limits": {"mimo-v2.5": 1048576}}) + st2 = st.model_copy(update={"improve_ledger_dir": str(tmp_path / "elsewhere"), + "improve_input_context_limits": {"mimo-v2.5": 1}}) seen_env: list[dict] = [] def spy(**kw): @@ -434,6 +436,7 @@ def spy(**kw): now=T0 + 200, judge_llm=PairedJudge(0.8, 0.6), governor=holder) assert seen_env and all(e["IMPROVE_LEDGER_DIR"] == str(holder.dir.parent) and e["IMPROVE_BUDGET_USD_WEEK"] == "50.0" for e in seen_env) + assert all(json.loads(e["IMPROVE_INPUT_CONTEXT_LIMITS"]) == {"mimo-v2.5": 1048576} for e in seen_env) # an arm that tries to override the engine's own configuration is refused at registration with pytest.raises(exps.ExperimentError, match="engine's own configuration"): exps.register(store, st, tmp_path / "ledger", workflow="pr-review.agent.review_diff", hypothesis="h",