Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,12 @@ export function ResearchAnalysisPanel({analysis: a}: {analysis: Controller}) {
const label = metricCopy[row.metric];
return <details key={row.metric}><summary><strong>{label?.[zh ? 0 : 1] ?? row.metric}</strong><span>{verdictLabel(row.metric, row.verdict, zh)}</span></summary><p className="analysis-question">{label?.[zh ? 2 : 3]}</p><p>{row.reason}</p>{citationButtons(row.citations)}</details>;
})}</div></details></section>)}
{a.citationBusy ? <p role="status">{zh ? "正在读取引用原文…" : "Reading cited source…"}</p> : null}
{a.citationError ? <p className="analysis-error" role="alert">{a.citationError}</p> : null}
{a.citation ? <section className="analysis-source" ref={a.sourcePanel} tabIndex={-1} aria-label={zh ? "引用原文" : "Cited original"} onKeyDown={event => {if (event.key === "Escape") {event.stopPropagation(); a.closeCitation();}}}><header><h4>{zh ? "引用原文" : "Cited original"}</h4><button type="button" onClick={a.closeCitation} aria-label={zh ? "关闭原文" : "Close source"}><X size={14} aria-hidden/></button></header><p>{a.citation.artifact} · {zh ? "行" : "lines"} {a.citation.first_line}–{a.citation.last_line}</p><pre>{a.citation.text}</pre></section> : null}
<details className="analysis-method"><summary>{zh ? "来源、方法、覆盖与分析成本" : "Sources, method, coverage and analysis cost"}</summary><p>{p.objective}</p><dl><dt>{zh ? "来源范围" : "Source scope"}</dt><dd>{p.scope.case_id} · {p.scope.run_id}<br/>{p.scope.window}</dd><dt>{zh ? "方法" : "Method"}</dt><dd>{result.method_version} · {zh ? "模型判断未校准" : "Uncalibrated model judgment"}</dd><dt>{zh ? "模型设置" : "Requested model"}</dt><dd>{p.config.model} / {p.config.effort} / {p.config.service_tier} · {zh ? "实际服务档位" : "Served tier"}: {p.job?.served_service_tier ?? (zh ? "未知" : "unknown")}</dd><dt>{zh ? "分析用量" : "Analysis usage"}</dt><dd>{p.job?.analysis_wall_seconds == null ? "—" : `${p.job.analysis_wall_seconds.toFixed(1)}s`} · {zh ? "输入" : "input"} {p.job?.usage?.input_tokens?.toLocaleString() ?? "—"} / {zh ? "输出" : "output"} {p.job?.usage?.output_tokens?.toLocaleString() ?? "—"} tokens · {zh ? "金额未采集,推理 token 已包含于输出" : "Dollar cost unmeasured; reasoning tokens are included in output"}</dd><dt>{zh ? "覆盖" : "Coverage"}</dt><dd>{p.episode_count} {zh ? "个固定片段" : "fixed episodes"} · {p.source_count} {zh ? "份来源" : "sources"} · {zh ? "总体片段数" : "Population"}: {p.selection.population_episodes ?? (zh ? "未知" : "unknown")}</dd><dt>{zh ? "取样规则" : "Selection"}</dt><dd>{p.selection.rule}</dd><dt>{zh ? "版本" : "Revision"}</dt><dd>{p.scope.revision}</dd></dl><ul>{result.limitations.map((text, i) => <li key={i}>{text}</li>)}</ul><p>{zh ? "引用行存在不等于解释正确;本页不提供独立科学校准结论。" : "A valid citation range does not establish entailment or independent scientific calibration."}</p></details>
</> : null}
{/* Bound native evidence is readable before any model assessment exists. */}
{a.citationBusy ? <p role="status">{zh ? "正在读取引用原文…" : "Reading cited source…"}</p> : null}
{a.citationError ? <p className="analysis-error" role="alert">{a.citationError}</p> : null}
{a.citation ? <section className="analysis-source" ref={a.sourcePanel} tabIndex={-1} aria-label={zh ? "引用原文" : "Cited original"} onKeyDown={event => {if (event.key === "Escape") {event.stopPropagation(); a.closeCitation();}}}><header><h4>{zh ? "引用原文" : "Cited original"}</h4><button type="button" onClick={a.closeCitation} aria-label={zh ? "关闭原文" : "Close source"}><X size={14} aria-hidden/></button></header><p>{a.citation.artifact} · {zh ? "行" : "lines"} {a.citation.first_line}–{a.citation.last_line}</p><pre>{a.citation.text}</pre></section> : null}
</> : null}
</div>;
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
import {ArrowUpRight, ChevronDown} from "lucide-react";
import {TRAJECTORY_METRICS} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js";
import type {AssessmentCitation, AssessmentVerdict, TrajectoryAssessment, TrajectoryEpisode, TrajectoryInput, TrajectoryMetric, TrajectorySolverUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js";
import type {TrajectoryNativeOutcomes} from "../../../../../../loopx/control_plane/capabilities/trajectory_native_outcomes.js";
import "./goal-research-progress.css";

// Read models from benchmark-toolkit. This view formats existing counts and
// assessments; it never scores, reclassifies an episode or combines runs.
export type ResearchProgressPack = {pack_id: string; title: string; scope: TrajectoryInput["scope"]; selection: TrajectoryInput["selection"];
episode_count: number; episodes?: TrajectoryEpisode[]; result?: TrajectoryAssessment | null; applicability?: "current" | "historical";
solver_usage?: {artifact: string; artifact_sha256: string; projection: TrajectorySolverUsage; applicability: "current" | "historical"; reason_code: string | null}};
job?: {status: string; attempt?: number; analysis_wall_seconds?: number | null;
usage?: {input_tokens?: number; output_tokens?: number} | null; config?: {model: string; effort: string; service_tier: string}} | null;
solver_usage?: {artifact: string; artifact_sha256: string; projection: TrajectorySolverUsage; applicability: "current" | "historical"; reason_code: string | null};
native_outcomes?: {artifact: string; artifact_sha256: string; projection: TrajectoryNativeOutcomes; applicability: "current" | "historical"; reason_code: string | null}};
export const metricCopy: Record<TrajectoryMetric, [string, string, string, string]> = {
goal_alignment: ["目标关联", "Goal alignment", "干预是否关联原目标的真实瓶颈", "Does the intervention address a bottleneck of the original objective?"],
benefit_test: ["收益检验", "Benefit test", "是否真正检验原目标收益,而非只完成前置诊断", "Was original-objective utility tested, beyond diagnostic prerequisites?"],
Expand All @@ -25,6 +29,7 @@ export function verdictLabel(metric: TrajectoryMetric, verdict: AssessmentVerdic

export function ResearchProgress({pack: p, packs, selectedId, error, zh, onSelect, onCitation}: {pack: ResearchProgressPack | null; packs: Pick<ResearchProgressPack, "pack_id" | "title">[]; selectedId: string; error: string; zh: boolean; onSelect: (id: string) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) {
const measured = p?.solver_usage, usage = measured?.projection, result = p?.result;
const native = p?.native_outcomes, outcomes = native?.projection;
const number = (n: number) => new Intl.NumberFormat(zh ? "zh-CN" : "en", {maximumFractionDigits: 1}).format(n);
const percent = (n: number) => new Intl.NumberFormat(zh ? "zh-CN" : "en", {style: "percent", maximumFractionDigits: 0}).format(n);
const duration = (s: number) => `${number(s / 60)} ${zh ? "分钟" : "min"}`;
Expand Down Expand Up @@ -67,10 +72,28 @@ export function ResearchProgress({pack: p, packs, selectedId, error, zh, onSelec
<dt>{zh ? "题目 / 运行" : "Case / run"}</dt><dd>{p.scope.case_id}<br/>{p.scope.run_id}</dd><dt>{zh ? "证据窗口" : "Evidence window"}</dt><dd>{p.scope.window}</dd><dt>{zh ? "取样规则" : "Selection rule"}</dt><dd>{p.selection.rule}</dd>
<dt>{zh ? "总体研究片段数" : "Population of episodes"}</dt><dd>{p.selection.population_episodes == null ? (zh ? "未知" : "Unknown") : number(p.selection.population_episodes)}</dd>
{result ? <><dt>{zh ? "分析方法" : "Analysis method"}</dt><dd>{result.method_version} · {zh ? "Agent 判读未校准" : "Uncalibrated Agent judgment"}</dd><dt>{zh ? "结果标识" : "Result key"}</dt><dd><code>{result.analysis_key}</code></dd><dt>{zh ? "分析限制" : "Limitations"}</dt><dd>{result.limitations.map((line, i) => <p key={i}>{line}</p>)}</dd></> : null}
<dt>{zh ? "来源版本" : "Source revision"}</dt><dd><code>{p.scope.revision}</code></dd><dt>{zh ? "原生收益" : "Native utility"}</dt><dd>{zh ? "此包尚未接入原生起点、保留最佳、最后有效值、达标时间/成本或增益面积投影。不能从模型判读或成本补算。" : "This pack has no native incumbent, retained-best, final-valid, threshold time/cost or gain-area projection. Model judgments and usage cannot supply them."}</dd>
<dt>{zh ? "来源版本" : "Source revision"}</dt><dd><code>{p.scope.revision}</code></dd>
<dt>{zh ? "原生结果" : "Native outcomes"}</dt><dd>{outcomes ? <>
<p>{outcomes.metric.name} · {outcomes.metric.unit} · {outcomes.metric.higher_is_better ? (zh ? "越高越好" : "higher is better") : (zh ? "越低越好" : "lower is better")}</p>
<p>{outcomes.start} — {outcomes.end} · {outcomes.complete ? (zh ? "声明为完整前缀" : "Declared complete prefix") : (zh ? "部分窗口,起点未知" : "Partial window; initial unknown")} · {outcomes.valid_count} / {outcomes.sample_count} {zh ? "项有效结果" : "valid results"}</p>
<dl>{([ ["initial", "首次提交", "Initial submission"], ["best_valid", "窗口内最佳有效值", "Best valid in capture"], ["retained", "原生保留结果", "Native retained result"], ["final_valid", "最后有效提交", "Final valid submission"] ] as const).map(([key, cn, en]) => {
const point = outcomes[key];
return <div key={key}><dt>{zh ? cn : en}</dt><dd>{point ? <>{String(point.value)} · {point.candidate_ref} <button type="button" onClick={event => onCitation(point.citation, event.currentTarget)}>{zh ? "查看原始证据" : "Read original evidence"}</button></> : (zh ? "— 未知,不记零" : "— Unknown, not zero")}</dd></div>;
})}</dl>
{native?.applicability === "historical" ? <p role="status">{zh ? "原生来源已变化或不可读;仅保留历史捕获,当前结果未核验。" : "Native source changed or is unavailable; retained historical capture, current results unverified."}</p> : null}
<p>{zh ? "原生测量与分析器判断分开。有效性和保留选择来自提供方,不认证研究因果收益;用量窗口独立,不补算达标成本或全程账单。" : "Native measurements are separate from analyzer judgments. Validity and retention are provider-recorded, not causal benefit. Usage has its own window; threshold and total-study costs are not inferred."}</p>
</> : (zh ? "— 尚无同范围的原生结果捕获;不能从分析器判断或用量补算。" : "— No native capture for this scope; analyzer judgments and usage cannot supply it.")}</dd>
</dl></details>
<details className="research-usage"><summary>{zh ? "用量与成本" : "Usage and cost"}{usage ? ` · ${zh ? "捕获跨度" : "Captured elapsed"} ${duration(usage.elapsed_seconds)}` : ` · ${zh ? "尚无完整用量捕获" : "No complete usage capture"}`}{measured?.applicability === "historical" ? (zh ? " · 历史捕获" : " · Historical capture") : ""}</summary>
<p>{zh ? "只读捕获,窗口可能与以上判读片段及下方 Explore 历史不同。" : "Read-only capture; its window can differ from the judgments above and Explore history below."}</p>
<dl className="research-usage-basis">
<dt>{zh ? "分析尝试开销" : "Analysis attempt cost"}</dt><dd>{p.job?.usage ? <>
{zh ? "第" : "Attempt"} {p.job.attempt ?? "—"} · {zh ? "输入" : "input"} {p.job.usage.input_tokens == null ? "—" : number(p.job.usage.input_tokens)} · {zh ? "输出" : "output"} {p.job.usage.output_tokens == null ? "—" : number(p.job.usage.output_tokens)} tokens
{p.job.config ? ` · ${p.job.config.model}` : ""}
{p.job.analysis_wall_seconds != null ? ` · ${duration(p.job.analysis_wall_seconds)}` : ""}
</> : (zh ? "— 本次分析尝试未测得,不记零" : "— No measurement for this analysis attempt; not zero")}</dd>
<dt>{zh ? "合计边界" : "Aggregation boundary"}</dt><dd>{zh ? "分析仅显示当前回执的本次尝试;不含更早失败、恢复、评测或操作者开销。不与求解捕获盲目相加,金额及完整研究成本仍未知。" : "Analysis is this attempt's receipt only; earlier failures, recovery, evaluation and operator work are excluded. It is not blindly added to solver capture. Monetary and complete study costs remain unknown."}</dd>
</dl>
<dl className="research-progress-values">
<div><dt>{zh ? "捕获窗口跨度" : "Captured elapsed time"}</dt><dd>{usage ? duration(usage.elapsed_seconds) : "—"}</dd><small>{usage ? `${number(usage.completed_turns)} ${zh ? "轮完整记录,不是全程总成本" : "complete Turns, not total study cost"}` : (zh ? "尚无完整用量捕获" : "No complete usage capture")}</small></div>
<div><dt>{zh ? "未缓存输入" : "Uncached input"}</dt><dd>{usage ? number(usage.uncached_input_tokens) : "—"}</dd><small>{usage ? "tokens" : (zh ? "未测,不记零" : "Unmeasured, not zero")}</small></div>
Expand Down
10 changes: 10 additions & 0 deletions docs/architecture/rfcs/goal-workspace-surface-v0.md
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,16 @@ independent scientific calibration or completion of V12. Host registration
remains preparation outside the ordinary reading journey. CLI preparation uses
the producer's existing interface; no new interactive Lark path is qualified.

The follow-up native-readback proposal extends that same trusted registration
with an optional provider outcome capture, scoped to the exact case/run/revision.
The existing source disclosure separates first submission, best valid in capture,
native-retained and final valid values, with original citations and historical/
partial/unknown states. Submission order, not feedback arrival, identifies the
candidate. This is not another scorer or automatic solver feedback. Captured
solver usage and the current analysis attempt's receipt stay separate; full-study
cost and causal benefit remain unknown. No first-viewport/hero/CTA change is
introduced; six uncalibrated process judgments retain their existing role.

## 12. Open decisions

| # | Decision | Owner | Options | Recommendation | Needed before |
Expand Down
6 changes: 6 additions & 0 deletions docs/architecture/rfcs/goal-workspace-surface-v0.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,12 @@ producer 接入一次显式分析、取消、重试和有引用的结果阅读
任意 run 自动准备、独立科学校准或 V12 完成。host 注册仍是普通阅读旅程之外的准备,
CLI 准备复用 producer 既有接口;本切片未验收新增飞书交互入口。

后续原生读回提案在同一可信注册入口增加可选 provider 结果捕获,固定题目、运行和
来源版本。既有来源展开分别展示首次提交、捕获内最佳有效值、原生保留和最后有效提交,
含原文引用及历史/部分/未知状态;候选按提交顺序而非反馈到达排序。不新增评分器或
自动 solver 回灌。求解捕获与本次分析尝试的回执开销分别保留,完整研究成本及因果收益
仍未知;首屏、hero、CTA 不变,六项未校准的过程判断维持原有角色。

## 12. 未决事项

| # | 事项 | Owner | 选项 | 建议 | 需在何时前决定 |
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
71 changes: 71 additions & 0 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,77 @@ journey or qualify the model assessor.

## Measured solver session cost (optional)

### Native outcome readback (optional, separate from analyzer judgments)

The trusted provider/Agent may add `--native-outcomes-json native-outcomes.json`
to the existing `benchmark trajectory-pack-register` operation. The file is
relative to its already authorized `--source-root`; the renderer cannot register
paths or invent measurements. Reading it does not execute analysis or change a
solver, evaluator, retained archive or study board. Omitting it preserves the
previous pack identity and response shape.

The provider-exported JSON has this local diagnostic contract:

```json
{
"schema_version": "trajectory_native_outcomes_v1",
"scope": {"goal_id":"solver", "case_id":"case", "run_id":"run", "revision":"fixed", "window":"declared capture"},
"start": "2026-01-01T00:00:00Z",
"end": "2026-01-01T00:10:00Z",
"complete": true,
"metric": {"name":"error", "unit":"points", "higher_is_better":false},
"retained_result_id": "result-1",
"samples": [{
"submission_index":1, "result_id":"result-1", "candidate_ref":"candidate-a",
"submitted_at":"2026-01-01T00:01:00Z", "observed_at":"2026-01-01T00:02:00Z",
"valid":true, "value":2,
"citation":{"source_id":"native-source", "first_line":1, "last_line":1}
}]
}
```

Every scope field must equal the registered assessment input. Source IDs and
line ranges must belong to that input; measurement/candidate IDs are opaque.
UTC timestamps must name real calendar instants (including leap-day validity),
not dates that a JavaScript parser silently normalizes into another window.
The source must contain provider-recorded released outcomes, not assessor
ratings or hidden evaluator material. The input curator/provider remains
responsible for the claim's truth and validity. `complete` declares prefix
coverage, not official integrity or score countability. A complete prefix has
contiguous submission indices starting at one. An invalid record uses
`valid:false,value:null`; zero is a valid measured value, never a missing value.
`retained_result_id:null` means retention is unknown, not that the mathematically
best value was adopted.

The existing typed benchmark-toolkit prepare owner returns a separate bounded
`native_outcomes` projection: first valid submission-one value when the prefix
is complete, best valid measurement within the capture under the explicit native
metric direction, provider-selected retained result, and final valid submission.
These are distinct: delayed feedback does not promote an older submission to
the final candidate, and a better measured score does not override retention or
guardrails. No generic score, monetary estimate or causal benefit is derived.
The original analysis key/model input is unchanged; native capture provenance
instead versions the pack identity. Changed/unavailable files leave historical
captured values visible with their warning, rather than borrowing another pack.

The existing Goal research source disclosure renders those facts and opens only
their bound original citations, even before model analysis exists. Solver
capture, native outcome and analyzed-episode windows remain separate. The cost
disclosure also shows the current analysis attempt's observed receipt usage;
earlier failures, native evaluation, recovery, operator costs and unmeasured
windows are not silently combined into a full-study bill. This does not complete
threshold-cost, gain-area, matched-study or scientific qualification.

Synthetic packaged native readback (not benchmark findings):

![Native outcome source disclosure](../development/assets/research-analysis/native-outcomes-desktop.png)

[Narrow-screen source disclosure](../development/assets/research-analysis/native-outcomes-mobile.png).
The existing six-judgment first viewport is unchanged; these facts live in its
existing source disclosure, with costs in their separate disclosure.

### Solver capture registration

A trusted pack curator can bind a **complete captured Codex session** separately
from the model evidence. The existing register command accepts:

Expand Down
Loading
Loading